2024-04-26T04:40:47.699 INFO:root:teuthology version: 0.0.1.dev269+g9481b1d 2024-04-26T04:40:47.699 DEBUG:teuthology.run:Teuthology command: teuthology --owner scheduled_teuthology@teuthology --description upgrade/quincy-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} --verbose --archive /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674591 --name teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi -- /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674591/orig.config.yaml 2024-04-26T04:40:47.767 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-26T04:40:47.868 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674591 branch: squid description: upgrade/quincy-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' first_in_suite: false job_id: '7674591' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run basic cls tests for rbd ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run basic import/export cli tests for rbd ' name: teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: squid ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: squid sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a owner: scheduled_teuthology@teuthology priority: 100 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' seed: 8320 sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 10/32 suite: upgrade suite_branch: squid suite_path: /home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a targets: smithi012.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLQp1eksonPbHSsFkISdJS1mr85/NCEJlpWQ7569jVuIEjfQ4+x+AcGk7ceUAMcYqkHkjRQfjcNAymm0jFgno8I= smithi116.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPcs/l1BxH4L87ucyG+asq1KBgaJ3eiUQ1VuYA1pzhZi8/EK1kYC2o5jiQ7S2SuSMTa9ZrbPD6jIhFj/bdLTNwM= tasks: - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17 timestamp: 2024-04-26_02:08:13 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-26T04:40:47.868 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa; will attempt to use it 2024-04-26T04:40:47.869 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa/tasks 2024-04-26T04:40:47.869 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-26T04:40:47.871 INFO:teuthology.task.internal:Checking packages... 2024-04-26T04:40:47.895 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'b22e2ebdeb24376882b7bda2a7329c8cccc2276a' 2024-04-26T04:40:47.895 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T04:40:47.895 INFO:teuthology.packaging:ref: None 2024-04-26T04:40:47.895 INFO:teuthology.packaging:tag: None 2024-04-26T04:40:47.895 INFO:teuthology.packaging:branch: squid 2024-04-26T04:40:47.895 INFO:teuthology.packaging:sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:40:47.895 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-04-26T04:40:48.202 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-2447.gb22e2ebd 2024-04-26T04:40:48.203 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-26T04:40:48.214 INFO:teuthology.task.internal:no buildpackages task found 2024-04-26T04:40:48.214 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-26T04:40:48.360 INFO:teuthology.task.internal:Saving configuration 2024-04-26T04:40:48.382 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-26T04:40:48.442 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-26T04:40:48.468 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi012.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674591', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-26 04:35:51.182766', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLQp1eksonPbHSsFkISdJS1mr85/NCEJlpWQ7569jVuIEjfQ4+x+AcGk7ceUAMcYqkHkjRQfjcNAymm0jFgno8I='} 2024-04-26T04:40:48.495 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi116.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674591', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-26 04:35:51.184183', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPcs/l1BxH4L87ucyG+asq1KBgaJ3eiUQ1VuYA1pzhZi8/EK1kYC2o5jiQ7S2SuSMTa9ZrbPD6jIhFj/bdLTNwM='} 2024-04-26T04:40:48.496 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-26T04:40:48.533 INFO:teuthology.task.internal:roles: ubuntu@smithi012.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-26T04:40:48.533 INFO:teuthology.task.internal:roles: ubuntu@smithi116.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-26T04:40:48.534 INFO:teuthology.run_tasks:Running task console_log... 2024-04-26T04:40:48.720 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f146beaf0d0>, signals=[15]) 2024-04-26T04:40:48.720 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-26T04:40:48.728 INFO:teuthology.task.internal:Opening connections... 2024-04-26T04:40:48.728 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi012.front.sepia.ceph.com 2024-04-26T04:40:48.730 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi012.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-26T04:40:48.839 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi116.front.sepia.ceph.com 2024-04-26T04:40:48.840 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi116.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-26T04:40:48.914 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-26T04:40:48.923 DEBUG:teuthology.orchestra.run.smithi012:> uname -m 2024-04-26T04:40:48.942 INFO:teuthology.orchestra.run.smithi012.stdout:x86_64 2024-04-26T04:40:48.942 DEBUG:teuthology.orchestra.run.smithi012:> cat /etc/os-release 2024-04-26T04:40:49.000 INFO:teuthology.orchestra.run.smithi012.stdout:NAME="CentOS Stream" 2024-04-26T04:40:49.000 INFO:teuthology.orchestra.run.smithi012.stdout:VERSION="9" 2024-04-26T04:40:49.000 INFO:teuthology.orchestra.run.smithi012.stdout:ID="centos" 2024-04-26T04:40:49.001 INFO:teuthology.orchestra.run.smithi012.stdout:ID_LIKE="rhel fedora" 2024-04-26T04:40:49.001 INFO:teuthology.orchestra.run.smithi012.stdout:VERSION_ID="9" 2024-04-26T04:40:49.001 INFO:teuthology.orchestra.run.smithi012.stdout:PLATFORM_ID="platform:el9" 2024-04-26T04:40:49.001 INFO:teuthology.orchestra.run.smithi012.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-26T04:40:49.001 INFO:teuthology.orchestra.run.smithi012.stdout:ANSI_COLOR="0;31" 2024-04-26T04:40:49.001 INFO:teuthology.orchestra.run.smithi012.stdout:LOGO="fedora-logo-icon" 2024-04-26T04:40:49.001 INFO:teuthology.orchestra.run.smithi012.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-26T04:40:49.001 INFO:teuthology.orchestra.run.smithi012.stdout:HOME_URL="https://centos.org/" 2024-04-26T04:40:49.001 INFO:teuthology.orchestra.run.smithi012.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-26T04:40:49.001 INFO:teuthology.orchestra.run.smithi012.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-26T04:40:49.001 INFO:teuthology.orchestra.run.smithi012.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-26T04:40:49.002 INFO:teuthology.lock.ops:Updating smithi012.front.sepia.ceph.com on lock server 2024-04-26T04:40:49.097 DEBUG:teuthology.orchestra.run.smithi116:> uname -m 2024-04-26T04:40:49.115 INFO:teuthology.orchestra.run.smithi116.stdout:x86_64 2024-04-26T04:40:49.115 DEBUG:teuthology.orchestra.run.smithi116:> cat /etc/os-release 2024-04-26T04:40:49.173 INFO:teuthology.orchestra.run.smithi116.stdout:NAME="CentOS Stream" 2024-04-26T04:40:49.173 INFO:teuthology.orchestra.run.smithi116.stdout:VERSION="9" 2024-04-26T04:40:49.173 INFO:teuthology.orchestra.run.smithi116.stdout:ID="centos" 2024-04-26T04:40:49.173 INFO:teuthology.orchestra.run.smithi116.stdout:ID_LIKE="rhel fedora" 2024-04-26T04:40:49.173 INFO:teuthology.orchestra.run.smithi116.stdout:VERSION_ID="9" 2024-04-26T04:40:49.173 INFO:teuthology.orchestra.run.smithi116.stdout:PLATFORM_ID="platform:el9" 2024-04-26T04:40:49.173 INFO:teuthology.orchestra.run.smithi116.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-26T04:40:49.173 INFO:teuthology.orchestra.run.smithi116.stdout:ANSI_COLOR="0;31" 2024-04-26T04:40:49.173 INFO:teuthology.orchestra.run.smithi116.stdout:LOGO="fedora-logo-icon" 2024-04-26T04:40:49.173 INFO:teuthology.orchestra.run.smithi116.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-26T04:40:49.173 INFO:teuthology.orchestra.run.smithi116.stdout:HOME_URL="https://centos.org/" 2024-04-26T04:40:49.174 INFO:teuthology.orchestra.run.smithi116.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-26T04:40:49.174 INFO:teuthology.orchestra.run.smithi116.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-26T04:40:49.174 INFO:teuthology.orchestra.run.smithi116.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-26T04:40:49.174 INFO:teuthology.lock.ops:Updating smithi116.front.sepia.ceph.com on lock server 2024-04-26T04:40:49.372 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-26T04:40:49.383 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-26T04:40:49.446 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-26T04:40:49.447 DEBUG:teuthology.orchestra.run.smithi012:> test '!' -e /home/ubuntu/cephtest 2024-04-26T04:40:49.453 DEBUG:teuthology.orchestra.run.smithi116:> test '!' -e /home/ubuntu/cephtest 2024-04-26T04:40:49.469 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-26T04:40:49.478 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-26T04:40:49.478 DEBUG:teuthology.orchestra.run.smithi012:> test -z $(ls -A /var/lib/ceph) 2024-04-26T04:40:49.510 DEBUG:teuthology.orchestra.run.smithi116:> test -z $(ls -A /var/lib/ceph) 2024-04-26T04:40:49.535 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-26T04:40:49.585 INFO:teuthology.run_tasks:Running task kernel... 2024-04-26T04:40:49.602 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-26T04:40:49.603 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-26T04:40:49.603 DEBUG:teuthology.orchestra.run.smithi012:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-26T04:40:49.604 DEBUG:teuthology.orchestra.run.smithi116:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-26T04:40:49.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T04:40:49.620 DEBUG:teuthology.orchestra.run.smithi116:> uname -r 2024-04-26T04:40:49.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T04:40:49.622 DEBUG:teuthology.orchestra.run.smithi012:> uname -r 2024-04-26T04:40:49.678 INFO:teuthology.orchestra.run.smithi116.stdout:5.14.0-437.el9.x86_64 2024-04-26T04:40:49.678 INFO:teuthology.task.kernel:Running kernel on smithi116: 5.14.0-437.el9.x86_64 2024-04-26T04:40:49.678 DEBUG:teuthology.orchestra.run.smithi116:> sudo yum install -y kernel 2024-04-26T04:40:49.680 INFO:teuthology.orchestra.run.smithi012.stdout:5.14.0-437.el9.x86_64 2024-04-26T04:40:49.680 INFO:teuthology.task.kernel:Running kernel on smithi012: 5.14.0-437.el9.x86_64 2024-04-26T04:40:49.680 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum install -y kernel 2024-04-26T04:40:51.989 INFO:teuthology.orchestra.run.smithi012.stdout:CentOS Stream 9 - BaseOS 12 MB/s | 8.0 MB 00:00 2024-04-26T04:40:52.164 INFO:teuthology.orchestra.run.smithi116.stdout:CentOS Stream 9 - BaseOS 9.8 MB/s | 8.0 MB 00:00 2024-04-26T04:40:53.973 INFO:teuthology.orchestra.run.smithi116.stdout:CentOS Stream 9 - AppStream 25 MB/s | 19 MB 00:00 2024-04-26T04:40:54.106 INFO:teuthology.orchestra.run.smithi012.stdout:CentOS Stream 9 - AppStream 19 MB/s | 19 MB 00:01 2024-04-26T04:40:58.282 INFO:teuthology.orchestra.run.smithi116.stdout:CentOS Stream 9 - CRB 9.1 MB/s | 6.2 MB 00:00 2024-04-26T04:40:58.340 INFO:teuthology.orchestra.run.smithi012.stdout:CentOS Stream 9 - CRB 11 MB/s | 6.2 MB 00:00 2024-04-26T04:40:59.705 INFO:teuthology.orchestra.run.smithi116.stdout:CentOS Stream 9 - Extras packages 57 kB/s | 16 kB 00:00 2024-04-26T04:40:59.838 INFO:teuthology.orchestra.run.smithi012.stdout:CentOS Stream 9 - Extras packages 54 kB/s | 16 kB 00:00 2024-04-26T04:41:01.648 INFO:teuthology.orchestra.run.smithi012.stdout:Extra Packages for Enterprise Linux 12 MB/s | 21 MB 00:01 2024-04-26T04:41:06.839 INFO:teuthology.orchestra.run.smithi012.stdout:lab-extras 24 kB/s | 1.7 kB 00:00 2024-04-26T04:41:08.804 INFO:teuthology.orchestra.run.smithi012.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-26T04:41:08.804 INFO:teuthology.orchestra.run.smithi012.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-04-26T04:41:08.887 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-04-26T04:41:08.888 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2024-04-26T04:41:08.888 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-04-26T04:41:09.009 DEBUG:teuthology.orchestra.run.smithi012:> echo no | sudo yum reinstall kernel || true 2024-04-26T04:41:09.529 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:00:03 ago on Fri 26 Apr 2024 04:41:06 AM UTC. 2024-04-26T04:41:09.666 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-04-26T04:41:09.671 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:41:09.671 INFO:teuthology.orchestra.run.smithi012.stdout: Package Architecture Version Repository Size 2024-04-26T04:41:09.671 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:41:09.671 INFO:teuthology.orchestra.run.smithi012.stdout:Reinstalling: 2024-04-26T04:41:09.671 INFO:teuthology.orchestra.run.smithi012.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-26T04:41:09.671 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:41:09.671 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-04-26T04:41:09.671 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:41:09.671 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:41:09.671 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 255 k 2024-04-26T04:41:09.671 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 0 2024-04-26T04:41:09.672 INFO:teuthology.orchestra.run.smithi012.stderr:Operation aborted. 2024-04-26T04:41:09.731 INFO:teuthology.orchestra.run.smithi012.stdout:Is this ok [y/N]: 2024-04-26T04:41:09.731 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum reinstall -y kernel || true 2024-04-26T04:41:10.247 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:00:04 ago on Fri 26 Apr 2024 04:41:06 AM UTC. 2024-04-26T04:41:10.368 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-04-26T04:41:10.369 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:41:10.369 INFO:teuthology.orchestra.run.smithi012.stdout: Package Architecture Version Repository Size 2024-04-26T04:41:10.369 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:41:10.369 INFO:teuthology.orchestra.run.smithi012.stdout:Reinstalling: 2024-04-26T04:41:10.369 INFO:teuthology.orchestra.run.smithi012.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-26T04:41:10.370 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:41:10.370 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-04-26T04:41:10.370 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:41:10.370 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:41:10.370 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 255 k 2024-04-26T04:41:10.370 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 0 2024-04-26T04:41:10.370 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-04-26T04:41:10.574 INFO:teuthology.orchestra.run.smithi012.stdout:kernel-5.14.0-437.el9.x86_64.rpm 1.8 MB/s | 255 kB 00:00 2024-04-26T04:41:10.574 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:41:10.574 INFO:teuthology.orchestra.run.smithi012.stdout:Total 1.2 MB/s | 255 kB 00:00 2024-04-26T04:41:10.597 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-04-26T04:41:10.788 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-04-26T04:41:10.788 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-04-26T04:41:10.793 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-04-26T04:41:10.793 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-04-26T04:41:10.932 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-04-26T04:41:11.000 INFO:teuthology.orchestra.run.smithi012.stdout: Reinstalling : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-26T04:41:11.000 INFO:teuthology.orchestra.run.smithi012.stdout: Cleanup : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T04:41:11.651 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T04:41:11.651 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-26T04:41:12.339 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T04:41:12.339 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:41:12.339 INFO:teuthology.orchestra.run.smithi012.stdout:Reinstalled: 2024-04-26T04:41:12.339 INFO:teuthology.orchestra.run.smithi012.stdout: kernel-5.14.0-437.el9.x86_64 2024-04-26T04:41:12.339 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:41:12.339 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-04-26T04:41:12.521 DEBUG:teuthology.orchestra.run.smithi012:> rpm -q kernel | sort -rV | head -n 1 2024-04-26T04:41:12.570 INFO:teuthology.orchestra.run.smithi012.stdout:kernel-5.14.0-437.el9.x86_64 2024-04-26T04:41:12.570 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-437.el9.x86_64 2024-04-26T04:41:12.570 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-26T04:41:12.570 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-26T04:41:12.571 DEBUG:teuthology.orchestra.run.smithi012:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-26T04:41:12.600 INFO:teuthology.orchestra.run.smithi012.stdout:ttyS1 2024-04-26T04:41:12.628 DEBUG:teuthology.parallel:result is None 2024-04-26T04:41:19.266 INFO:teuthology.orchestra.run.smithi116.stdout:Extra Packages for Enterprise Linux 1.1 MB/s | 21 MB 00:19 2024-04-26T04:41:24.354 INFO:teuthology.orchestra.run.smithi116.stdout:lab-extras 23 kB/s | 1.7 kB 00:00 2024-04-26T04:41:26.302 INFO:teuthology.orchestra.run.smithi116.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-26T04:41:26.302 INFO:teuthology.orchestra.run.smithi116.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-04-26T04:41:26.392 INFO:teuthology.orchestra.run.smithi116.stdout:Dependencies resolved. 2024-04-26T04:41:26.393 INFO:teuthology.orchestra.run.smithi116.stdout:Nothing to do. 2024-04-26T04:41:26.393 INFO:teuthology.orchestra.run.smithi116.stdout:Complete! 2024-04-26T04:41:26.506 DEBUG:teuthology.orchestra.run.smithi116:> echo no | sudo yum reinstall kernel || true 2024-04-26T04:41:27.002 INFO:teuthology.orchestra.run.smithi116.stdout:Last metadata expiration check: 0:00:03 ago on Fri 26 Apr 2024 04:41:24 AM UTC. 2024-04-26T04:41:27.120 INFO:teuthology.orchestra.run.smithi116.stdout:Dependencies resolved. 2024-04-26T04:41:27.138 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:41:27.138 INFO:teuthology.orchestra.run.smithi116.stdout: Package Architecture Version Repository Size 2024-04-26T04:41:27.138 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:41:27.138 INFO:teuthology.orchestra.run.smithi116.stdout:Reinstalling: 2024-04-26T04:41:27.139 INFO:teuthology.orchestra.run.smithi116.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-26T04:41:27.139 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:41:27.139 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction Summary 2024-04-26T04:41:27.139 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:41:27.139 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:41:27.139 INFO:teuthology.orchestra.run.smithi116.stdout:Total download size: 255 k 2024-04-26T04:41:27.139 INFO:teuthology.orchestra.run.smithi116.stdout:Installed size: 0 2024-04-26T04:41:27.139 INFO:teuthology.orchestra.run.smithi116.stderr:Operation aborted. 2024-04-26T04:41:27.204 INFO:teuthology.orchestra.run.smithi116.stdout:Is this ok [y/N]: 2024-04-26T04:41:27.205 DEBUG:teuthology.orchestra.run.smithi116:> sudo yum reinstall -y kernel || true 2024-04-26T04:41:27.696 INFO:teuthology.orchestra.run.smithi116.stdout:Last metadata expiration check: 0:00:03 ago on Fri 26 Apr 2024 04:41:24 AM UTC. 2024-04-26T04:41:27.811 INFO:teuthology.orchestra.run.smithi116.stdout:Dependencies resolved. 2024-04-26T04:41:27.812 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:41:27.812 INFO:teuthology.orchestra.run.smithi116.stdout: Package Architecture Version Repository Size 2024-04-26T04:41:27.812 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:41:27.812 INFO:teuthology.orchestra.run.smithi116.stdout:Reinstalling: 2024-04-26T04:41:27.812 INFO:teuthology.orchestra.run.smithi116.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-26T04:41:27.812 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:41:27.812 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction Summary 2024-04-26T04:41:27.813 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:41:27.813 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:41:27.813 INFO:teuthology.orchestra.run.smithi116.stdout:Total download size: 255 k 2024-04-26T04:41:27.813 INFO:teuthology.orchestra.run.smithi116.stdout:Installed size: 0 2024-04-26T04:41:27.813 INFO:teuthology.orchestra.run.smithi116.stdout:Downloading Packages: 2024-04-26T04:41:28.014 INFO:teuthology.orchestra.run.smithi116.stdout:kernel-5.14.0-437.el9.x86_64.rpm 1.9 MB/s | 255 kB 00:00 2024-04-26T04:41:28.015 INFO:teuthology.orchestra.run.smithi116.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:41:28.015 INFO:teuthology.orchestra.run.smithi116.stdout:Total 1.2 MB/s | 255 kB 00:00 2024-04-26T04:41:28.032 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction check 2024-04-26T04:41:28.230 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction check succeeded. 2024-04-26T04:41:28.230 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction test 2024-04-26T04:41:28.235 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction test succeeded. 2024-04-26T04:41:28.235 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction 2024-04-26T04:41:28.357 INFO:teuthology.orchestra.run.smithi116.stdout: Preparing : 1/1 2024-04-26T04:41:28.449 INFO:teuthology.orchestra.run.smithi116.stdout: Reinstalling : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-26T04:41:28.449 INFO:teuthology.orchestra.run.smithi116.stdout: Cleanup : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T04:41:29.067 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T04:41:29.067 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-26T04:41:29.747 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T04:41:29.747 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:41:29.747 INFO:teuthology.orchestra.run.smithi116.stdout:Reinstalled: 2024-04-26T04:41:29.747 INFO:teuthology.orchestra.run.smithi116.stdout: kernel-5.14.0-437.el9.x86_64 2024-04-26T04:41:29.747 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:41:29.747 INFO:teuthology.orchestra.run.smithi116.stdout:Complete! 2024-04-26T04:41:29.932 DEBUG:teuthology.orchestra.run.smithi116:> rpm -q kernel | sort -rV | head -n 1 2024-04-26T04:41:29.981 INFO:teuthology.orchestra.run.smithi116.stdout:kernel-5.14.0-437.el9.x86_64 2024-04-26T04:41:29.981 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-437.el9.x86_64 2024-04-26T04:41:29.981 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-26T04:41:29.981 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-26T04:41:29.981 DEBUG:teuthology.orchestra.run.smithi116:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-26T04:41:30.014 INFO:teuthology.orchestra.run.smithi116.stdout:ttyS1 2024-04-26T04:41:30.035 DEBUG:teuthology.parallel:result is None 2024-04-26T04:41:30.035 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-26T04:41:30.043 INFO:teuthology.task.internal:Creating test directory... 2024-04-26T04:41:30.043 DEBUG:teuthology.orchestra.run.smithi012:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-26T04:41:30.045 DEBUG:teuthology.orchestra.run.smithi116:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-26T04:41:30.090 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-26T04:41:30.097 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-26T04:41:30.127 INFO:teuthology.task.internal:Creating archive directory... 2024-04-26T04:41:30.128 DEBUG:teuthology.orchestra.run.smithi012:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-26T04:41:30.130 DEBUG:teuthology.orchestra.run.smithi116:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-26T04:41:30.159 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-26T04:41:30.166 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-26T04:41:30.166 DEBUG:teuthology.orchestra.run.smithi012:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-26T04:41:30.197 DEBUG:teuthology.orchestra.run.smithi116:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-26T04:41:30.230 INFO:teuthology.orchestra.run.smithi116.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-26T04:41:30.238 INFO:teuthology.orchestra.run.smithi012.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-26T04:41:30.239 INFO:teuthology.orchestra.run.smithi116.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-26T04:41:30.249 INFO:teuthology.orchestra.run.smithi012.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-26T04:41:30.251 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-26T04:41:30.291 INFO:teuthology.task.internal:Configuring sudo... 2024-04-26T04:41:30.292 DEBUG:teuthology.orchestra.run.smithi012:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-26T04:41:30.294 DEBUG:teuthology.orchestra.run.smithi116:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-26T04:41:30.322 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-26T04:41:30.372 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-26T04:41:30.373 DEBUG:teuthology.orchestra.run.smithi012:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-26T04:41:30.375 DEBUG:teuthology.orchestra.run.smithi116:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-26T04:41:30.391 DEBUG:teuthology.orchestra.run.smithi012:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-26T04:41:30.477 DEBUG:teuthology.orchestra.run.smithi012:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-26T04:41:30.528 DEBUG:teuthology.orchestra.run.smithi012:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-26T04:41:30.608 DEBUG:teuthology.orchestra.run.smithi012:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-26T04:41:30.656 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-04-26T04:41:30.656 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-26T04:41:30.723 DEBUG:teuthology.orchestra.run.smithi116:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-26T04:41:30.765 DEBUG:teuthology.orchestra.run.smithi116:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-26T04:41:30.807 DEBUG:teuthology.orchestra.run.smithi116:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-26T04:41:30.889 DEBUG:teuthology.orchestra.run.smithi116:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-26T04:41:30.928 DEBUG:teuthology.orchestra.run.smithi116:> set -ex 2024-04-26T04:41:30.929 DEBUG:teuthology.orchestra.run.smithi116:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-26T04:41:30.991 DEBUG:teuthology.orchestra.run.smithi012:> sudo service rsyslog restart 2024-04-26T04:41:30.994 DEBUG:teuthology.orchestra.run.smithi116:> sudo service rsyslog restart 2024-04-26T04:41:31.030 INFO:teuthology.orchestra.run.smithi012.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-26T04:41:31.068 INFO:teuthology.orchestra.run.smithi116.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-26T04:41:31.547 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-26T04:41:31.555 INFO:teuthology.task.internal:Starting timer... 2024-04-26T04:41:31.555 INFO:teuthology.run_tasks:Running task pcp... 2024-04-26T04:41:31.580 INFO:teuthology.run_tasks:Running task selinux... 2024-04-26T04:41:31.626 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-04-26T04:41:31.626 DEBUG:teuthology.orchestra.run.smithi012:> sudo service auditd rotate 2024-04-26T04:41:31.678 INFO:teuthology.orchestra.run.smithi012.stdout:Rotating logs: 2024-04-26T04:41:31.680 DEBUG:teuthology.orchestra.run.smithi116:> sudo service auditd rotate 2024-04-26T04:41:31.753 INFO:teuthology.orchestra.run.smithi116.stdout:Rotating logs: 2024-04-26T04:41:31.755 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-26T04:41:31.757 DEBUG:teuthology.orchestra.run.smithi012:> /usr/sbin/getenforce 2024-04-26T04:41:31.796 INFO:teuthology.orchestra.run.smithi012.stdout:Permissive 2024-04-26T04:41:31.796 DEBUG:teuthology.orchestra.run.smithi116:> /usr/sbin/getenforce 2024-04-26T04:41:31.832 INFO:teuthology.orchestra.run.smithi116.stdout:Permissive 2024-04-26T04:41:31.832 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi012.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi116.front.sepia.ceph.com': 'permissive'} 2024-04-26T04:41:31.832 DEBUG:teuthology.orchestra.run.smithi012:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-26T04:41:31.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T04:41:31.865 DEBUG:teuthology.orchestra.run.smithi116:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-26T04:41:31.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T04:41:31.897 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-26T04:41:31.897 DEBUG:teuthology.orchestra.run.smithi012:> sudo /usr/sbin/setenforce permissive 2024-04-26T04:41:31.931 DEBUG:teuthology.orchestra.run.smithi116:> sudo /usr/sbin/setenforce permissive 2024-04-26T04:41:31.985 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-26T04:41:31.995 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-26T04:41:32.011 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-26T04:41:32.012 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi012.front.sepia.ceph.com,smithi116.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-26T04:45:04.226 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi012.front.sepia.ceph.com'), Remote(name='ubuntu@smithi116.front.sepia.ceph.com')] 2024-04-26T04:45:04.228 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi012.front.sepia.ceph.com' 2024-04-26T04:45:04.229 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi012.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-26T04:45:04.310 DEBUG:teuthology.orchestra.run.smithi012:> true 2024-04-26T04:45:04.346 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi012.front.sepia.ceph.com' 2024-04-26T04:45:04.347 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi116.front.sepia.ceph.com' 2024-04-26T04:45:04.347 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi116.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-26T04:45:04.421 DEBUG:teuthology.orchestra.run.smithi116:> true 2024-04-26T04:45:04.497 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi116.front.sepia.ceph.com' 2024-04-26T04:45:04.497 INFO:teuthology.run_tasks:Running task clock... 2024-04-26T04:45:04.510 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-04-26T04:45:04.510 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-26T04:45:04.510 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-26T04:45:04.513 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-26T04:45:04.513 DEBUG:teuthology.orchestra.run.smithi116:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-26T04:45:04.544 INFO:teuthology.orchestra.run.smithi012.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-26T04:45:04.559 INFO:teuthology.orchestra.run.smithi012.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-26T04:45:04.564 INFO:teuthology.orchestra.run.smithi116.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-26T04:45:04.577 INFO:teuthology.orchestra.run.smithi116.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-26T04:45:04.585 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: ntpd: command not found 2024-04-26T04:45:04.596 INFO:teuthology.orchestra.run.smithi012.stdout:506 Cannot talk to daemon 2024-04-26T04:45:04.599 INFO:teuthology.orchestra.run.smithi116.stderr:sudo: ntpd: command not found 2024-04-26T04:45:04.609 INFO:teuthology.orchestra.run.smithi116.stdout:506 Cannot talk to daemon 2024-04-26T04:45:04.611 INFO:teuthology.orchestra.run.smithi012.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-26T04:45:04.622 INFO:teuthology.orchestra.run.smithi116.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-26T04:45:04.625 INFO:teuthology.orchestra.run.smithi012.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-26T04:45:04.635 INFO:teuthology.orchestra.run.smithi116.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-26T04:45:04.680 INFO:teuthology.orchestra.run.smithi012.stderr:bash: line 1: ntpq: command not found 2024-04-26T04:45:04.682 INFO:teuthology.orchestra.run.smithi012.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-26T04:45:04.683 INFO:teuthology.orchestra.run.smithi012.stdout:=============================================================================== 2024-04-26T04:45:04.683 INFO:teuthology.orchestra.run.smithi012.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:45:04.683 INFO:teuthology.orchestra.run.smithi012.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:45:04.683 INFO:teuthology.orchestra.run.smithi012.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:45:04.683 INFO:teuthology.orchestra.run.smithi012.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:45:04.691 INFO:teuthology.orchestra.run.smithi116.stderr:bash: line 1: ntpq: command not found 2024-04-26T04:45:04.696 INFO:teuthology.orchestra.run.smithi116.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-26T04:45:04.696 INFO:teuthology.orchestra.run.smithi116.stdout:=============================================================================== 2024-04-26T04:45:04.696 INFO:teuthology.orchestra.run.smithi116.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:45:04.696 INFO:teuthology.orchestra.run.smithi116.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:45:04.696 INFO:teuthology.orchestra.run.smithi116.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:45:04.696 INFO:teuthology.orchestra.run.smithi116.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:45:04.696 INFO:teuthology.run_tasks:Running task install... 2024-04-26T04:45:04.708 DEBUG:teuthology.task.install:project ceph 2024-04-26T04:45:04.708 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'b22e2ebdeb24376882b7bda2a7329c8cccc2276a'}} 2024-04-26T04:45:04.708 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'b22e2ebdeb24376882b7bda2a7329c8cccc2276a'} 2024-04-26T04:45:04.708 INFO:teuthology.task.install:Using flavor: default 2024-04-26T04:45:04.714 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-04-26T04:45:04.714 INFO:teuthology.task.install:extra packages: [] 2024-04-26T04:45:04.715 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'b22e2ebdeb24376882b7bda2a7329c8cccc2276a', 'tag': None, 'wait_for_package': False} 2024-04-26T04:45:04.715 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T04:45:04.715 INFO:teuthology.packaging:ref: None 2024-04-26T04:45:04.715 INFO:teuthology.packaging:tag: None 2024-04-26T04:45:04.715 INFO:teuthology.packaging:branch: quincy 2024-04-26T04:45:04.715 INFO:teuthology.packaging:sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:45:04.715 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-26T04:45:04.717 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'b22e2ebdeb24376882b7bda2a7329c8cccc2276a', 'tag': None, 'wait_for_package': False} 2024-04-26T04:45:04.717 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T04:45:04.718 INFO:teuthology.packaging:ref: None 2024-04-26T04:45:04.718 INFO:teuthology.packaging:tag: None 2024-04-26T04:45:04.718 INFO:teuthology.packaging:branch: quincy 2024-04-26T04:45:04.718 INFO:teuthology.packaging:sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:45:04.718 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-26T04:45:04.958 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/ 2024-04-26T04:45:04.958 INFO:teuthology.task.install.rpm:Package version is 17.2.7-904.gd8c6b0a3 2024-04-26T04:45:05.023 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/ 2024-04-26T04:45:05.024 INFO:teuthology.task.install.rpm:Package version is 17.2.7-904.gd8c6b0a3 2024-04-26T04:45:05.135 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-04-26T04:45:05.135 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-04-26T04:45:05.135 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-04-26T04:45:05.163 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-04-26T04:45:05.163 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T04:45:05.163 INFO:teuthology.packaging:ref: None 2024-04-26T04:45:05.163 INFO:teuthology.packaging:tag: None 2024-04-26T04:45:05.163 INFO:teuthology.packaging:branch: quincy 2024-04-26T04:45:05.163 INFO:teuthology.packaging:sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:45:05.163 DEBUG:teuthology.orchestra.run.smithi012:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-04-26T04:45:05.196 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-04-26T04:45:05.196 DEBUG:teuthology.orchestra.run.smithi116:> set -ex 2024-04-26T04:45:05.196 DEBUG:teuthology.orchestra.run.smithi116:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-04-26T04:45:05.222 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-04-26T04:45:05.222 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T04:45:05.222 INFO:teuthology.packaging:ref: None 2024-04-26T04:45:05.222 INFO:teuthology.packaging:tag: None 2024-04-26T04:45:05.222 INFO:teuthology.packaging:branch: quincy 2024-04-26T04:45:05.222 INFO:teuthology.packaging:sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:45:05.222 DEBUG:teuthology.orchestra.run.smithi116:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-04-26T04:45:05.230 DEBUG:teuthology.orchestra.run.smithi012:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-04-26T04:45:05.287 DEBUG:teuthology.orchestra.run.smithi116:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-04-26T04:45:05.306 DEBUG:teuthology.orchestra.run.smithi012:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-04-26T04:45:05.360 DEBUG:teuthology.orchestra.run.smithi116:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-04-26T04:45:05.371 INFO:teuthology.orchestra.run.smithi012.stdout:check_obsoletes = 1 2024-04-26T04:45:05.372 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum clean all 2024-04-26T04:45:05.423 INFO:teuthology.orchestra.run.smithi116.stdout:check_obsoletes = 1 2024-04-26T04:45:05.425 DEBUG:teuthology.orchestra.run.smithi116:> sudo yum clean all 2024-04-26T04:45:05.607 INFO:teuthology.orchestra.run.smithi012.stdout:48 files removed 2024-04-26T04:45:05.634 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-radosgw 2024-04-26T04:45:05.649 INFO:teuthology.orchestra.run.smithi116.stdout:48 files removed 2024-04-26T04:45:05.675 DEBUG:teuthology.orchestra.run.smithi116:> sudo yum -y install ceph-radosgw 2024-04-26T04:45:06.292 INFO:teuthology.orchestra.run.smithi012.stdout:ceph packages for x86_64 200 kB/s | 77 kB 00:00 2024-04-26T04:45:06.344 INFO:teuthology.orchestra.run.smithi116.stdout:ceph packages for x86_64 187 kB/s | 77 kB 00:00 2024-04-26T04:45:06.678 INFO:teuthology.orchestra.run.smithi116.stdout:ceph noarch packages 37 kB/s | 11 kB 00:00 2024-04-26T04:45:06.684 INFO:teuthology.orchestra.run.smithi012.stdout:ceph noarch packages 31 kB/s | 11 kB 00:00 2024-04-26T04:45:07.045 INFO:teuthology.orchestra.run.smithi116.stdout:ceph source packages 5.4 kB/s | 1.8 kB 00:00 2024-04-26T04:45:07.060 INFO:teuthology.orchestra.run.smithi012.stdout:ceph source packages 5.3 kB/s | 1.8 kB 00:00 2024-04-26T04:45:07.770 INFO:teuthology.orchestra.run.smithi012.stdout:CentOS Stream 9 - BaseOS 12 MB/s | 8.0 MB 00:00 2024-04-26T04:45:07.781 INFO:teuthology.orchestra.run.smithi116.stdout:CentOS Stream 9 - BaseOS 11 MB/s | 8.0 MB 00:00 2024-04-26T04:45:09.658 INFO:teuthology.orchestra.run.smithi116.stdout:CentOS Stream 9 - AppStream 24 MB/s | 19 MB 00:00 2024-04-26T04:45:09.969 INFO:teuthology.orchestra.run.smithi012.stdout:CentOS Stream 9 - AppStream 18 MB/s | 19 MB 00:01 2024-04-26T04:45:14.817 INFO:teuthology.orchestra.run.smithi116.stdout:CentOS Stream 9 - CRB 4.1 MB/s | 6.2 MB 00:01 2024-04-26T04:45:15.281 INFO:teuthology.orchestra.run.smithi012.stdout:CentOS Stream 9 - CRB 3.9 MB/s | 6.2 MB 00:01 2024-04-26T04:45:16.440 INFO:teuthology.orchestra.run.smithi116.stdout:CentOS Stream 9 - Extras packages 36 kB/s | 16 kB 00:00 2024-04-26T04:45:16.753 INFO:teuthology.orchestra.run.smithi012.stdout:CentOS Stream 9 - Extras packages 67 kB/s | 16 kB 00:00 2024-04-26T04:45:17.527 INFO:teuthology.orchestra.run.smithi116.stdout:Extra Packages for Enterprise Linux 22 MB/s | 21 MB 00:00 2024-04-26T04:45:18.690 INFO:teuthology.orchestra.run.smithi012.stdout:Extra Packages for Enterprise Linux 12 MB/s | 21 MB 00:01 2024-04-26T04:45:22.633 INFO:teuthology.orchestra.run.smithi116.stdout:lab-extras 13 kB/s | 1.7 kB 00:00 2024-04-26T04:45:24.013 INFO:teuthology.orchestra.run.smithi012.stdout:lab-extras 47 kB/s | 1.7 kB 00:00 2024-04-26T04:45:24.758 INFO:teuthology.orchestra.run.smithi116.stdout:Dependencies resolved. 2024-04-26T04:45:24.760 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:45:24.760 INFO:teuthology.orchestra.run.smithi116.stdout: Package Arch Version Repository Size 2024-04-26T04:45:24.760 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:45:24.760 INFO:teuthology.orchestra.run.smithi116.stdout:Installing: 2024-04-26T04:45:24.761 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-radosgw x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 14 M 2024-04-26T04:45:24.761 INFO:teuthology.orchestra.run.smithi116.stdout:Upgrading: 2024-04-26T04:45:24.761 INFO:teuthology.orchestra.run.smithi116.stdout: librados2 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 3.1 M 2024-04-26T04:45:24.761 INFO:teuthology.orchestra.run.smithi116.stdout: librbd1 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 2.9 M 2024-04-26T04:45:24.761 INFO:teuthology.orchestra.run.smithi116.stdout:Installing dependencies: 2024-04-26T04:45:24.761 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-base x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 5.0 M 2024-04-26T04:45:24.761 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-common x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 22 M 2024-04-26T04:45:24.761 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-selinux x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 25 k 2024-04-26T04:45:24.761 INFO:teuthology.orchestra.run.smithi116.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-04-26T04:45:24.761 INFO:teuthology.orchestra.run.smithi116.stdout: libcephfs2 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 637 k 2024-04-26T04:45:24.761 INFO:teuthology.orchestra.run.smithi116.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-04-26T04:45:24.761 INFO:teuthology.orchestra.run.smithi116.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-04-26T04:45:24.761 INFO:teuthology.orchestra.run.smithi116.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-04-26T04:45:24.762 INFO:teuthology.orchestra.run.smithi116.stdout: libradosstriper1 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 457 k 2024-04-26T04:45:24.762 INFO:teuthology.orchestra.run.smithi116.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-04-26T04:45:24.762 INFO:teuthology.orchestra.run.smithi116.stdout: librgw2 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 4.9 M 2024-04-26T04:45:24.762 INFO:teuthology.orchestra.run.smithi116.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-04-26T04:45:24.762 INFO:teuthology.orchestra.run.smithi116.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-04-26T04:45:24.762 INFO:teuthology.orchestra.run.smithi116.stdout: python3-ceph-argparse x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 45 k 2024-04-26T04:45:24.762 INFO:teuthology.orchestra.run.smithi116.stdout: python3-ceph-common x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 117 k 2024-04-26T04:45:24.762 INFO:teuthology.orchestra.run.smithi116.stdout: python3-cephfs x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 156 k 2024-04-26T04:45:24.762 INFO:teuthology.orchestra.run.smithi116.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-04-26T04:45:24.762 INFO:teuthology.orchestra.run.smithi116.stdout: python3-rados x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 320 k 2024-04-26T04:45:24.762 INFO:teuthology.orchestra.run.smithi116.stdout: python3-rbd x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 293 k 2024-04-26T04:45:24.762 INFO:teuthology.orchestra.run.smithi116.stdout: python3-rgw x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 100 k 2024-04-26T04:45:24.763 INFO:teuthology.orchestra.run.smithi116.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-04-26T04:45:24.763 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:45:24.763 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction Summary 2024-04-26T04:45:24.763 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:45:24.763 INFO:teuthology.orchestra.run.smithi116.stdout:Install 22 Packages 2024-04-26T04:45:24.763 INFO:teuthology.orchestra.run.smithi116.stdout:Upgrade 2 Packages 2024-04-26T04:45:24.763 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:45:24.763 INFO:teuthology.orchestra.run.smithi116.stdout:Total download size: 58 M 2024-04-26T04:45:24.764 INFO:teuthology.orchestra.run.smithi116.stdout:Downloading Packages: 2024-04-26T04:45:26.231 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-04-26T04:45:26.233 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:45:26.233 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2024-04-26T04:45:26.233 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:45:26.233 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-04-26T04:45:26.234 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-radosgw x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 14 M 2024-04-26T04:45:26.234 INFO:teuthology.orchestra.run.smithi012.stdout:Upgrading: 2024-04-26T04:45:26.234 INFO:teuthology.orchestra.run.smithi012.stdout: librados2 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 3.1 M 2024-04-26T04:45:26.234 INFO:teuthology.orchestra.run.smithi012.stdout: librbd1 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 2.9 M 2024-04-26T04:45:26.234 INFO:teuthology.orchestra.run.smithi012.stdout:Installing dependencies: 2024-04-26T04:45:26.234 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-base x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 5.0 M 2024-04-26T04:45:26.234 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-common x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 22 M 2024-04-26T04:45:26.234 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-selinux x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 25 k 2024-04-26T04:45:26.234 INFO:teuthology.orchestra.run.smithi012.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-04-26T04:45:26.234 INFO:teuthology.orchestra.run.smithi012.stdout: libcephfs2 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 637 k 2024-04-26T04:45:26.234 INFO:teuthology.orchestra.run.smithi012.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-04-26T04:45:26.234 INFO:teuthology.orchestra.run.smithi012.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-04-26T04:45:26.235 INFO:teuthology.orchestra.run.smithi012.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-04-26T04:45:26.235 INFO:teuthology.orchestra.run.smithi012.stdout: libradosstriper1 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 457 k 2024-04-26T04:45:26.235 INFO:teuthology.orchestra.run.smithi012.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-04-26T04:45:26.235 INFO:teuthology.orchestra.run.smithi012.stdout: librgw2 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 4.9 M 2024-04-26T04:45:26.235 INFO:teuthology.orchestra.run.smithi012.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-04-26T04:45:26.235 INFO:teuthology.orchestra.run.smithi012.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-04-26T04:45:26.235 INFO:teuthology.orchestra.run.smithi012.stdout: python3-ceph-argparse x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 45 k 2024-04-26T04:45:26.235 INFO:teuthology.orchestra.run.smithi012.stdout: python3-ceph-common x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 117 k 2024-04-26T04:45:26.235 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cephfs x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 156 k 2024-04-26T04:45:26.235 INFO:teuthology.orchestra.run.smithi012.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-04-26T04:45:26.235 INFO:teuthology.orchestra.run.smithi012.stdout: python3-rados x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 320 k 2024-04-26T04:45:26.235 INFO:teuthology.orchestra.run.smithi012.stdout: python3-rbd x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 293 k 2024-04-26T04:45:26.235 INFO:teuthology.orchestra.run.smithi012.stdout: python3-rgw x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 100 k 2024-04-26T04:45:26.236 INFO:teuthology.orchestra.run.smithi012.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-04-26T04:45:26.236 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:45:26.236 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-04-26T04:45:26.236 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:45:26.236 INFO:teuthology.orchestra.run.smithi012.stdout:Install 22 Packages 2024-04-26T04:45:26.236 INFO:teuthology.orchestra.run.smithi012.stdout:Upgrade 2 Packages 2024-04-26T04:45:26.236 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:45:26.237 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 58 M 2024-04-26T04:45:26.237 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-04-26T04:45:26.667 INFO:teuthology.orchestra.run.smithi116.stdout:(1/24): ceph-base-17.2.7-904.gd8c6b0a3.el9.x86_ 3.9 MB/s | 5.0 MB 00:01 2024-04-26T04:45:26.734 INFO:teuthology.orchestra.run.smithi116.stdout:(2/24): ceph-selinux-17.2.7-904.gd8c6b0a3.el9.x 378 kB/s | 25 kB 00:00 2024-04-26T04:45:26.909 INFO:teuthology.orchestra.run.smithi116.stdout:(3/24): libcephfs2-17.2.7-904.gd8c6b0a3.el9.x86 3.6 MB/s | 637 kB 00:00 2024-04-26T04:45:27.226 INFO:teuthology.orchestra.run.smithi116.stdout:(4/24): ceph-common-17.2.7-904.gd8c6b0a3.el9.x8 12 MB/s | 22 MB 00:01 2024-04-26T04:45:27.251 INFO:teuthology.orchestra.run.smithi116.stdout:(5/24): libradosstriper1-17.2.7-904.gd8c6b0a3.e 1.3 MB/s | 457 kB 00:00 2024-04-26T04:45:27.335 INFO:teuthology.orchestra.run.smithi116.stdout:(6/24): python3-ceph-argparse-17.2.7-904.gd8c6b 543 kB/s | 45 kB 00:00 2024-04-26T04:45:27.460 INFO:teuthology.orchestra.run.smithi116.stdout:(7/24): python3-ceph-common-17.2.7-904.gd8c6b0a 941 kB/s | 117 kB 00:00 2024-04-26T04:45:27.578 INFO:teuthology.orchestra.run.smithi116.stdout:(8/24): librgw2-17.2.7-904.gd8c6b0a3.el9.x86_64 14 MB/s | 4.9 MB 00:00 2024-04-26T04:45:27.603 INFO:teuthology.orchestra.run.smithi116.stdout:(9/24): python3-cephfs-17.2.7-904.gd8c6b0a3.el9 1.1 MB/s | 156 kB 00:00 2024-04-26T04:45:27.628 INFO:teuthology.orchestra.run.smithi116.stdout:(10/24): python3-rados-17.2.7-904.gd8c6b0a3.el9 6.3 MB/s | 320 kB 00:00 2024-04-26T04:45:27.686 INFO:teuthology.orchestra.run.smithi116.stdout:(11/24): python3-rgw-17.2.7-904.gd8c6b0a3.el9.x 1.7 MB/s | 100 kB 00:00 2024-04-26T04:45:27.712 INFO:teuthology.orchestra.run.smithi116.stdout:(12/24): python3-rbd-17.2.7-904.gd8c6b0a3.el9.x 2.6 MB/s | 293 kB 00:00 2024-04-26T04:45:27.896 INFO:teuthology.orchestra.run.smithi116.stdout:(13/24): librabbitmq-0.11.0-7.el9.x86_64.rpm 247 kB/s | 45 kB 00:00 2024-04-26T04:45:27.938 INFO:teuthology.orchestra.run.smithi116.stdout:(14/24): libpmemobj-1.12.1-1.el9.x86_64.rpm 639 kB/s | 160 kB 00:00 2024-04-26T04:45:28.055 INFO:teuthology.orchestra.run.smithi116.stdout:(15/24): librdkafka-1.6.1-102.el9.x86_64.rpm 4.1 MB/s | 662 kB 00:00 2024-04-26T04:45:28.080 INFO:teuthology.orchestra.run.smithi116.stdout:(16/24): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.0 MB/s | 292 kB 00:00 2024-04-26T04:45:28.122 INFO:teuthology.orchestra.run.smithi116.stdout:(17/24): python3-prettytable-0.7.2-27.el9.noarc 637 kB/s | 42 kB 00:00 2024-04-26T04:45:28.299 INFO:teuthology.orchestra.run.smithi116.stdout:(18/24): liboath-2.6.7-2.el9.x86_64.rpm 274 kB/s | 49 kB 00:00 2024-04-26T04:45:28.391 INFO:teuthology.orchestra.run.smithi116.stdout:(19/24): gperftools-libs-2.9.1-2.el9.x86_64.rpm 993 kB/s | 309 kB 00:00 2024-04-26T04:45:28.417 INFO:teuthology.orchestra.run.smithi116.stdout:(20/24): libunwind-1.6.2-1.el9.x86_64.rpm 576 kB/s | 67 kB 00:00 2024-04-26T04:45:28.626 INFO:teuthology.orchestra.run.smithi116.stdout:(21/24): ceph-radosgw-17.2.7-904.gd8c6b0a3.el9. 4.4 MB/s | 14 MB 00:03 2024-04-26T04:45:28.663 INFO:teuthology.orchestra.run.smithi012.stdout:(1/24): ceph-common-17.2.7-904.gd8c6b0a3.el9.x8 13 MB/s | 22 MB 00:01 2024-04-26T04:45:28.709 INFO:teuthology.orchestra.run.smithi116.stdout:(22/24): thrift-0.15.0-2.el9.x86_64.rpm 5.0 MB/s | 1.6 MB 00:00 2024-04-26T04:45:28.714 INFO:teuthology.orchestra.run.smithi012.stdout:(2/24): ceph-selinux-17.2.7-904.gd8c6b0a3.el9.x 506 kB/s | 25 kB 00:00 2024-04-26T04:45:28.855 INFO:teuthology.orchestra.run.smithi012.stdout:(3/24): libcephfs2-17.2.7-904.gd8c6b0a3.el9.x86 4.4 MB/s | 637 kB 00:00 2024-04-26T04:45:28.981 INFO:teuthology.orchestra.run.smithi012.stdout:(4/24): libradosstriper1-17.2.7-904.gd8c6b0a3.e 3.6 MB/s | 457 kB 00:00 2024-04-26T04:45:29.135 INFO:teuthology.orchestra.run.smithi116.stdout:(23/24): librados2-17.2.7-904.gd8c6b0a3.el9.x86 4.3 MB/s | 3.1 MB 00:00 2024-04-26T04:45:29.184 INFO:teuthology.orchestra.run.smithi116.stdout:(24/24): librbd1-17.2.7-904.gd8c6b0a3.el9.x86_6 5.3 MB/s | 2.9 MB 00:00 2024-04-26T04:45:29.189 INFO:teuthology.orchestra.run.smithi116.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:45:29.189 INFO:teuthology.orchestra.run.smithi116.stdout:Total 13 MB/s | 58 MB 00:04 2024-04-26T04:45:29.249 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction check 2024-04-26T04:45:29.282 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction check succeeded. 2024-04-26T04:45:29.331 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction test 2024-04-26T04:45:29.439 INFO:teuthology.orchestra.run.smithi012.stdout:(5/24): librgw2-17.2.7-904.gd8c6b0a3.el9.x86_64 11 MB/s | 4.9 MB 00:00 2024-04-26T04:45:29.490 INFO:teuthology.orchestra.run.smithi012.stdout:(6/24): python3-ceph-argparse-17.2.7-904.gd8c6b 907 kB/s | 45 kB 00:00 2024-04-26T04:45:29.565 INFO:teuthology.orchestra.run.smithi012.stdout:(7/24): python3-ceph-common-17.2.7-904.gd8c6b0a 1.5 MB/s | 117 kB 00:00 2024-04-26T04:45:29.640 INFO:teuthology.orchestra.run.smithi012.stdout:(8/24): python3-cephfs-17.2.7-904.gd8c6b0a3.el9 2.0 MB/s | 156 kB 00:00 2024-04-26T04:45:29.717 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction test succeeded. 2024-04-26T04:45:29.717 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction 2024-04-26T04:45:29.724 INFO:teuthology.orchestra.run.smithi012.stdout:(9/24): python3-rados-17.2.7-904.gd8c6b0a3.el9. 3.8 MB/s | 320 kB 00:00 2024-04-26T04:45:29.791 INFO:teuthology.orchestra.run.smithi012.stdout:(10/24): python3-rbd-17.2.7-904.gd8c6b0a3.el9.x 4.3 MB/s | 293 kB 00:00 2024-04-26T04:45:29.866 INFO:teuthology.orchestra.run.smithi012.stdout:(11/24): python3-rgw-17.2.7-904.gd8c6b0a3.el9.x 1.3 MB/s | 100 kB 00:00 2024-04-26T04:45:29.983 INFO:teuthology.orchestra.run.smithi012.stdout:(12/24): libpmemobj-1.12.1-1.el9.x86_64.rpm 1.3 MB/s | 160 kB 00:00 2024-04-26T04:45:30.016 INFO:teuthology.orchestra.run.smithi012.stdout:(13/24): librabbitmq-0.11.0-7.el9.x86_64.rpm 1.3 MB/s | 45 kB 00:00 2024-04-26T04:45:30.075 INFO:teuthology.orchestra.run.smithi012.stdout:(14/24): librdkafka-1.6.1-102.el9.x86_64.rpm 11 MB/s | 662 kB 00:00 2024-04-26T04:45:30.126 INFO:teuthology.orchestra.run.smithi012.stdout:(15/24): lttng-ust-2.12.0-6.el9.x86_64.rpm 5.7 MB/s | 292 kB 00:00 2024-04-26T04:45:30.159 INFO:teuthology.orchestra.run.smithi012.stdout:(16/24): python3-prettytable-0.7.2-27.el9.noarc 1.2 MB/s | 42 kB 00:00 2024-04-26T04:45:30.954 INFO:teuthology.orchestra.run.smithi116.stdout: Preparing : 1/1 2024-04-26T04:45:31.347 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/26 2024-04-26T04:45:31.412 INFO:teuthology.orchestra.run.smithi116.stdout: Upgrading : librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/26 2024-04-26T04:45:31.486 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/26 2024-04-26T04:45:31.564 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/26 2024-04-26T04:45:31.588 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/26 2024-04-26T04:45:31.617 INFO:teuthology.orchestra.run.smithi012.stdout:(17/24): gperftools-libs-2.9.1-2.el9.x86_64.rpm 212 kB/s | 309 kB 00:01 2024-04-26T04:45:31.675 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/26 2024-04-26T04:45:31.721 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : thrift-0.15.0-2.el9.x86_64 5/26 2024-04-26T04:45:31.760 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/26 2024-04-26T04:45:31.767 INFO:teuthology.orchestra.run.smithi012.stdout:(18/24): liboath-2.6.7-2.el9.x86_64.rpm 324 kB/s | 49 kB 00:00 2024-04-26T04:45:31.968 INFO:teuthology.orchestra.run.smithi012.stdout:(19/24): ceph-radosgw-17.2.7-904.gd8c6b0a3.el9. 2.8 MB/s | 14 MB 00:05 2024-04-26T04:45:31.993 INFO:teuthology.orchestra.run.smithi012.stdout:(20/24): libunwind-1.6.2-1.el9.x86_64.rpm 299 kB/s | 67 kB 00:00 2024-04-26T04:45:32.189 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 7/26 2024-04-26T04:45:32.224 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 8/26 2024-04-26T04:45:32.287 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 8/26 2024-04-26T04:45:32.315 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : liboath-2.6.7-2.el9.x86_64 9/26 2024-04-26T04:45:32.352 INFO:teuthology.orchestra.run.smithi012.stdout:(21/24): librados2-17.2.7-904.gd8c6b0a3.el9.x86 8.6 MB/s | 3.1 MB 00:00 2024-04-26T04:45:32.358 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-ceph-argparse-2:17.2.7-904.gd8c6b0a3.el9.x 10/26 2024-04-26T04:45:32.404 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 11/26 2024-04-26T04:45:32.465 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 12/26 2024-04-26T04:45:32.494 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 13/26 2024-04-26T04:45:32.558 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 13/26 2024-04-26T04:45:32.597 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/26 2024-04-26T04:45:32.627 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/26 2024-04-26T04:45:32.660 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 16/26 2024-04-26T04:45:32.768 INFO:teuthology.orchestra.run.smithi012.stdout:(22/24): librbd1-17.2.7-904.gd8c6b0a3.el9.x86_6 7.1 MB/s | 2.9 MB 00:00 2024-04-26T04:45:32.959 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/26 2024-04-26T04:45:33.006 INFO:teuthology.orchestra.run.smithi116.stdout: Upgrading : librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 18/26 2024-04-26T04:45:33.094 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 18/26 2024-04-26T04:45:33.168 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 19/26 2024-04-26T04:45:33.200 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86 20/26 2024-04-26T04:45:33.627 INFO:teuthology.orchestra.run.smithi012.stdout:(23/24): thrift-0.15.0-2.el9.x86_64.rpm 977 kB/s | 1.6 MB 00:01 2024-04-26T04:45:34.194 INFO:teuthology.orchestra.run.smithi012.stdout:(24/24): ceph-base-17.2.7-904.gd8c6b0a3.el9.x86 694 kB/s | 5.0 MB 00:07 2024-04-26T04:45:34.200 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:45:34.200 INFO:teuthology.orchestra.run.smithi012.stdout:Total 7.2 MB/s | 58 MB 00:07 2024-04-26T04:45:34.261 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-04-26T04:45:34.299 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-04-26T04:45:34.299 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-04-26T04:45:34.757 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-04-26T04:45:34.757 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-04-26T04:45:35.855 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 21/26 2024-04-26T04:45:35.888 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 21/26 2024-04-26T04:45:36.011 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-04-26T04:45:36.393 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 21/26 2024-04-26T04:45:36.414 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/26 2024-04-26T04:45:36.442 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 22/26 2024-04-26T04:45:36.454 INFO:teuthology.orchestra.run.smithi012.stdout: Upgrading : librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/26 2024-04-26T04:45:36.530 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/26 2024-04-26T04:45:36.542 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 22/26 2024-04-26T04:45:36.542 INFO:teuthology.orchestra.run.smithi116.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-04-26T04:45:36.542 INFO:teuthology.orchestra.run.smithi116.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-04-26T04:45:36.542 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:45:36.568 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 23/26 2024-04-26T04:45:36.611 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/26 2024-04-26T04:45:36.647 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/26 2024-04-26T04:45:36.736 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/26 2024-04-26T04:45:36.783 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : thrift-0.15.0-2.el9.x86_64 5/26 2024-04-26T04:45:36.811 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/26 2024-04-26T04:45:37.268 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 7/26 2024-04-26T04:45:37.292 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 8/26 2024-04-26T04:45:37.355 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 8/26 2024-04-26T04:45:37.393 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : liboath-2.6.7-2.el9.x86_64 9/26 2024-04-26T04:45:37.436 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-ceph-argparse-2:17.2.7-904.gd8c6b0a3.el9.x 10/26 2024-04-26T04:45:37.473 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 11/26 2024-04-26T04:45:37.536 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 12/26 2024-04-26T04:45:37.562 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 13/26 2024-04-26T04:45:37.611 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 13/26 2024-04-26T04:45:37.653 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/26 2024-04-26T04:45:37.690 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/26 2024-04-26T04:45:37.722 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 16/26 2024-04-26T04:45:38.030 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/26 2024-04-26T04:45:38.068 INFO:teuthology.orchestra.run.smithi012.stdout: Upgrading : librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 18/26 2024-04-26T04:45:38.163 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 18/26 2024-04-26T04:45:38.244 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 19/26 2024-04-26T04:45:38.270 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86 20/26 2024-04-26T04:45:40.756 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 21/26 2024-04-26T04:45:40.792 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 21/26 2024-04-26T04:45:41.331 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 21/26 2024-04-26T04:45:41.379 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 22/26 2024-04-26T04:45:41.480 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 22/26 2024-04-26T04:45:41.480 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-04-26T04:45:41.480 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-04-26T04:45:41.480 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:45:41.513 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 23/26 2024-04-26T04:45:45.974 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 23/26 2024-04-26T04:45:45.974 INFO:teuthology.orchestra.run.smithi116.stdout:skipping the directory /sys 2024-04-26T04:45:45.975 INFO:teuthology.orchestra.run.smithi116.stdout:skipping the directory /proc 2024-04-26T04:45:45.975 INFO:teuthology.orchestra.run.smithi116.stdout:skipping the directory /mnt 2024-04-26T04:45:45.975 INFO:teuthology.orchestra.run.smithi116.stdout:skipping the directory /var/tmp 2024-04-26T04:45:45.975 INFO:teuthology.orchestra.run.smithi116.stdout:skipping the directory /home 2024-04-26T04:45:45.975 INFO:teuthology.orchestra.run.smithi116.stdout:skipping the directory /root 2024-04-26T04:45:45.975 INFO:teuthology.orchestra.run.smithi116.stdout:skipping the directory /tmp 2024-04-26T04:45:45.975 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:45:47.268 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 24/26 2024-04-26T04:45:47.345 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 24/26 2024-04-26T04:45:47.346 INFO:teuthology.orchestra.run.smithi116.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:45:47.346 INFO:teuthology.orchestra.run.smithi116.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-04-26T04:45:47.346 INFO:teuthology.orchestra.run.smithi116.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-26T04:45:47.346 INFO:teuthology.orchestra.run.smithi116.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-26T04:45:47.346 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:45:47.347 INFO:teuthology.orchestra.run.smithi116.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-26T04:45:47.429 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-26T04:45:47.429 INFO:teuthology.orchestra.run.smithi116.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-26T04:45:48.739 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-26T04:45:48.739 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/26 2024-04-26T04:45:48.739 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/26 2024-04-26T04:45:48.739 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/26 2024-04-26T04:45:48.739 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/26 2024-04-26T04:45:48.739 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 5/26 2024-04-26T04:45:48.739 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 6/26 2024-04-26T04:45:48.739 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 7/26 2024-04-26T04:45:48.739 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-ceph-argparse-2:17.2.7-904.gd8c6b0a3.el9.x 8/26 2024-04-26T04:45:48.740 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86 9/26 2024-04-26T04:45:48.740 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 10/26 2024-04-26T04:45:48.740 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 11/26 2024-04-26T04:45:48.740 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 12/26 2024-04-26T04:45:48.740 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 13/26 2024-04-26T04:45:48.740 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 14/26 2024-04-26T04:45:48.740 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 15/26 2024-04-26T04:45:48.740 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 16/26 2024-04-26T04:45:48.740 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 17/26 2024-04-26T04:45:48.740 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 18/26 2024-04-26T04:45:48.740 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 19/26 2024-04-26T04:45:48.740 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 20/26 2024-04-26T04:45:48.741 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 21/26 2024-04-26T04:45:48.741 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 22/26 2024-04-26T04:45:48.741 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 23/26 2024-04-26T04:45:48.741 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 24/26 2024-04-26T04:45:48.741 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 25/26 2024-04-26T04:45:49.463 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 26/26 2024-04-26T04:45:49.463 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:45:49.463 INFO:teuthology.orchestra.run.smithi116.stdout:Upgraded: 2024-04-26T04:45:49.463 INFO:teuthology.orchestra.run.smithi116.stdout: librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:45:49.463 INFO:teuthology.orchestra.run.smithi116.stdout: librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:45:49.464 INFO:teuthology.orchestra.run.smithi116.stdout:Installed: 2024-04-26T04:45:49.464 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:45:49.464 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:45:49.464 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:45:49.464 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:45:49.464 INFO:teuthology.orchestra.run.smithi116.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-04-26T04:45:49.464 INFO:teuthology.orchestra.run.smithi116.stdout: libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:45:49.464 INFO:teuthology.orchestra.run.smithi116.stdout: liboath-2.6.7-2.el9.x86_64 2024-04-26T04:45:49.464 INFO:teuthology.orchestra.run.smithi116.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-04-26T04:45:49.464 INFO:teuthology.orchestra.run.smithi116.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-04-26T04:45:49.464 INFO:teuthology.orchestra.run.smithi116.stdout: libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:45:49.464 INFO:teuthology.orchestra.run.smithi116.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-04-26T04:45:49.464 INFO:teuthology.orchestra.run.smithi116.stdout: librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:45:49.465 INFO:teuthology.orchestra.run.smithi116.stdout: libunwind-1.6.2-1.el9.x86_64 2024-04-26T04:45:49.465 INFO:teuthology.orchestra.run.smithi116.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-04-26T04:45:49.465 INFO:teuthology.orchestra.run.smithi116.stdout: python3-ceph-argparse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:45:49.465 INFO:teuthology.orchestra.run.smithi116.stdout: python3-ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:45:49.465 INFO:teuthology.orchestra.run.smithi116.stdout: python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:45:49.465 INFO:teuthology.orchestra.run.smithi116.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-04-26T04:45:49.465 INFO:teuthology.orchestra.run.smithi116.stdout: python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:45:49.465 INFO:teuthology.orchestra.run.smithi116.stdout: python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:45:49.465 INFO:teuthology.orchestra.run.smithi116.stdout: python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:45:49.465 INFO:teuthology.orchestra.run.smithi116.stdout: thrift-0.15.0-2.el9.x86_64 2024-04-26T04:45:49.465 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:45:49.465 INFO:teuthology.orchestra.run.smithi116.stdout:Complete! 2024-04-26T04:45:49.741 DEBUG:teuthology.orchestra.run.smithi116:> sudo yum -y install ceph-test 2024-04-26T04:45:50.388 INFO:teuthology.orchestra.run.smithi116.stdout:Last metadata expiration check: 0:00:28 ago on Fri 26 Apr 2024 04:45:22 AM UTC. 2024-04-26T04:45:50.511 INFO:teuthology.orchestra.run.smithi116.stdout:Dependencies resolved. 2024-04-26T04:45:50.512 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:45:50.512 INFO:teuthology.orchestra.run.smithi116.stdout: Package Arch Version Repository Size 2024-04-26T04:45:50.512 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:45:50.512 INFO:teuthology.orchestra.run.smithi116.stdout:Installing: 2024-04-26T04:45:50.513 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-test x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 48 M 2024-04-26T04:45:50.513 INFO:teuthology.orchestra.run.smithi116.stdout:Installing dependencies: 2024-04-26T04:45:50.513 INFO:teuthology.orchestra.run.smithi116.stdout: libcephsqlite x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 163 k 2024-04-26T04:45:50.513 INFO:teuthology.orchestra.run.smithi116.stdout: socat x86_64 1.7.4.1-5.el9 appstream 305 k 2024-04-26T04:45:50.513 INFO:teuthology.orchestra.run.smithi116.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-04-26T04:45:50.513 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:45:50.513 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction Summary 2024-04-26T04:45:50.513 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:45:50.513 INFO:teuthology.orchestra.run.smithi116.stdout:Install 4 Packages 2024-04-26T04:45:50.513 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:45:50.514 INFO:teuthology.orchestra.run.smithi116.stdout:Total download size: 48 M 2024-04-26T04:45:50.514 INFO:teuthology.orchestra.run.smithi116.stdout:Installed size: 192 M 2024-04-26T04:45:50.514 INFO:teuthology.orchestra.run.smithi116.stdout:Downloading Packages: 2024-04-26T04:45:50.779 INFO:teuthology.orchestra.run.smithi116.stdout:(1/4): socat-1.7.4.1-5.el9.x86_64.rpm 1.5 MB/s | 305 kB 00:00 2024-04-26T04:45:50.838 INFO:teuthology.orchestra.run.smithi116.stdout:(2/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.1 MB/s | 64 kB 00:00 2024-04-26T04:45:50.888 INFO:teuthology.orchestra.run.smithi116.stdout:(3/4): libcephsqlite-17.2.7-904.gd8c6b0a3.el9.x 533 kB/s | 163 kB 00:00 2024-04-26T04:45:51.248 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 23/26 2024-04-26T04:45:51.248 INFO:teuthology.orchestra.run.smithi012.stdout:skipping the directory /sys 2024-04-26T04:45:51.248 INFO:teuthology.orchestra.run.smithi012.stdout:skipping the directory /proc 2024-04-26T04:45:51.248 INFO:teuthology.orchestra.run.smithi012.stdout:skipping the directory /mnt 2024-04-26T04:45:51.248 INFO:teuthology.orchestra.run.smithi012.stdout:skipping the directory /var/tmp 2024-04-26T04:45:51.248 INFO:teuthology.orchestra.run.smithi012.stdout:skipping the directory /home 2024-04-26T04:45:51.248 INFO:teuthology.orchestra.run.smithi012.stdout:skipping the directory /root 2024-04-26T04:45:51.248 INFO:teuthology.orchestra.run.smithi012.stdout:skipping the directory /tmp 2024-04-26T04:45:51.248 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:45:52.563 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 24/26 2024-04-26T04:45:52.634 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 24/26 2024-04-26T04:45:52.634 INFO:teuthology.orchestra.run.smithi012.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:45:52.634 INFO:teuthology.orchestra.run.smithi012.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-04-26T04:45:52.634 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-26T04:45:52.634 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-26T04:45:52.635 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:45:52.636 INFO:teuthology.orchestra.run.smithi012.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-26T04:45:52.725 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-26T04:45:52.725 INFO:teuthology.orchestra.run.smithi012.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-26T04:45:52.915 INFO:teuthology.orchestra.run.smithi116.stdout:(4/4): ceph-test-17.2.7-904.gd8c6b0a3.el9.x86_6 21 MB/s | 48 MB 00:02 2024-04-26T04:45:52.918 INFO:teuthology.orchestra.run.smithi116.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:45:52.919 INFO:teuthology.orchestra.run.smithi116.stdout:Total 20 MB/s | 48 MB 00:02 2024-04-26T04:45:52.945 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction check 2024-04-26T04:45:52.965 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction check succeeded. 2024-04-26T04:45:52.965 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction test 2024-04-26T04:45:53.327 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction test succeeded. 2024-04-26T04:45:53.328 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction 2024-04-26T04:45:53.848 INFO:teuthology.orchestra.run.smithi116.stdout: Preparing : 1/1 2024-04-26T04:45:53.934 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-04-26T04:45:53.940 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-26T04:45:53.940 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/26 2024-04-26T04:45:53.940 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/26 2024-04-26T04:45:53.940 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/26 2024-04-26T04:45:53.940 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/26 2024-04-26T04:45:53.940 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 5/26 2024-04-26T04:45:53.940 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 6/26 2024-04-26T04:45:53.941 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 7/26 2024-04-26T04:45:53.941 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-ceph-argparse-2:17.2.7-904.gd8c6b0a3.el9.x 8/26 2024-04-26T04:45:53.941 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86 9/26 2024-04-26T04:45:53.941 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 10/26 2024-04-26T04:45:53.941 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 11/26 2024-04-26T04:45:53.941 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 12/26 2024-04-26T04:45:53.941 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 13/26 2024-04-26T04:45:53.941 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 14/26 2024-04-26T04:45:53.941 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 15/26 2024-04-26T04:45:53.941 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 16/26 2024-04-26T04:45:53.941 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 17/26 2024-04-26T04:45:53.943 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 18/26 2024-04-26T04:45:53.943 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 19/26 2024-04-26T04:45:53.943 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 20/26 2024-04-26T04:45:53.943 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 21/26 2024-04-26T04:45:53.943 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 22/26 2024-04-26T04:45:53.943 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 23/26 2024-04-26T04:45:53.943 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 24/26 2024-04-26T04:45:53.943 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 25/26 2024-04-26T04:45:53.972 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 2/4 2024-04-26T04:45:54.000 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/4 2024-04-26T04:45:54.592 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 26/26 2024-04-26T04:45:54.593 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:45:54.593 INFO:teuthology.orchestra.run.smithi012.stdout:Upgraded: 2024-04-26T04:45:54.593 INFO:teuthology.orchestra.run.smithi012.stdout: librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:45:54.593 INFO:teuthology.orchestra.run.smithi012.stdout: librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:45:54.593 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-04-26T04:45:54.593 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:45:54.593 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:45:54.593 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:45:54.593 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:45:54.593 INFO:teuthology.orchestra.run.smithi012.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-04-26T04:45:54.593 INFO:teuthology.orchestra.run.smithi012.stdout: libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:45:54.593 INFO:teuthology.orchestra.run.smithi012.stdout: liboath-2.6.7-2.el9.x86_64 2024-04-26T04:45:54.594 INFO:teuthology.orchestra.run.smithi012.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-04-26T04:45:54.594 INFO:teuthology.orchestra.run.smithi012.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-04-26T04:45:54.594 INFO:teuthology.orchestra.run.smithi012.stdout: libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:45:54.594 INFO:teuthology.orchestra.run.smithi012.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-04-26T04:45:54.594 INFO:teuthology.orchestra.run.smithi012.stdout: librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:45:54.594 INFO:teuthology.orchestra.run.smithi012.stdout: libunwind-1.6.2-1.el9.x86_64 2024-04-26T04:45:54.594 INFO:teuthology.orchestra.run.smithi012.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-04-26T04:45:54.594 INFO:teuthology.orchestra.run.smithi012.stdout: python3-ceph-argparse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:45:54.594 INFO:teuthology.orchestra.run.smithi012.stdout: python3-ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:45:54.594 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:45:54.594 INFO:teuthology.orchestra.run.smithi012.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-04-26T04:45:54.594 INFO:teuthology.orchestra.run.smithi012.stdout: python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:45:54.594 INFO:teuthology.orchestra.run.smithi012.stdout: python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:45:54.595 INFO:teuthology.orchestra.run.smithi012.stdout: python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:45:54.595 INFO:teuthology.orchestra.run.smithi012.stdout: thrift-0.15.0-2.el9.x86_64 2024-04-26T04:45:54.595 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:45:54.595 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-04-26T04:45:54.839 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-test 2024-04-26T04:45:55.414 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:00:32 ago on Fri 26 Apr 2024 04:45:23 AM UTC. 2024-04-26T04:45:55.536 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-04-26T04:45:55.537 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:45:55.537 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2024-04-26T04:45:55.537 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:45:55.537 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-04-26T04:45:55.538 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-test x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 48 M 2024-04-26T04:45:55.538 INFO:teuthology.orchestra.run.smithi012.stdout:Installing dependencies: 2024-04-26T04:45:55.538 INFO:teuthology.orchestra.run.smithi012.stdout: libcephsqlite x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 163 k 2024-04-26T04:45:55.538 INFO:teuthology.orchestra.run.smithi012.stdout: socat x86_64 1.7.4.1-5.el9 appstream 305 k 2024-04-26T04:45:55.538 INFO:teuthology.orchestra.run.smithi012.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-04-26T04:45:55.538 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:45:55.538 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-04-26T04:45:55.538 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:45:55.538 INFO:teuthology.orchestra.run.smithi012.stdout:Install 4 Packages 2024-04-26T04:45:55.538 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:45:55.539 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 48 M 2024-04-26T04:45:55.539 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 192 M 2024-04-26T04:45:55.539 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-04-26T04:45:56.142 INFO:teuthology.orchestra.run.smithi012.stdout:(1/4): libcephsqlite-17.2.7-904.gd8c6b0a3.el9.x 481 kB/s | 163 kB 00:00 2024-04-26T04:45:56.933 INFO:teuthology.orchestra.run.smithi012.stdout:(2/4): socat-1.7.4.1-5.el9.x86_64.rpm 270 kB/s | 305 kB 00:01 2024-04-26T04:45:56.959 INFO:teuthology.orchestra.run.smithi012.stdout:(3/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 78 kB/s | 64 kB 00:00 2024-04-26T04:45:58.136 INFO:teuthology.orchestra.run.smithi012.stdout:(4/4): ceph-test-17.2.7-904.gd8c6b0a3.el9.x86_6 21 MB/s | 48 MB 00:02 2024-04-26T04:45:58.139 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:45:58.139 INFO:teuthology.orchestra.run.smithi012.stdout:Total 19 MB/s | 48 MB 00:02 2024-04-26T04:45:58.166 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-04-26T04:45:58.188 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-04-26T04:45:58.188 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-04-26T04:45:58.393 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/4 2024-04-26T04:45:58.472 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/4 2024-04-26T04:45:58.571 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-04-26T04:45:58.571 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-04-26T04:45:59.129 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-04-26T04:45:59.212 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-04-26T04:45:59.261 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 2/4 2024-04-26T04:45:59.287 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/4 2024-04-26T04:46:00.694 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/4 2024-04-26T04:46:00.694 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/4 2024-04-26T04:46:00.694 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/4 2024-04-26T04:46:00.694 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 3/4 2024-04-26T04:46:00.946 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-04-26T04:46:00.946 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:00.946 INFO:teuthology.orchestra.run.smithi116.stdout:Installed: 2024-04-26T04:46:00.946 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:00.946 INFO:teuthology.orchestra.run.smithi116.stdout: libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:00.947 INFO:teuthology.orchestra.run.smithi116.stdout: socat-1.7.4.1-5.el9.x86_64 2024-04-26T04:46:00.947 INFO:teuthology.orchestra.run.smithi116.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-04-26T04:46:00.947 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:00.947 INFO:teuthology.orchestra.run.smithi116.stdout:Complete! 2024-04-26T04:46:01.230 DEBUG:teuthology.orchestra.run.smithi116:> sudo yum -y install ceph 2024-04-26T04:46:01.739 INFO:teuthology.orchestra.run.smithi116.stdout:Last metadata expiration check: 0:00:39 ago on Fri 26 Apr 2024 04:45:22 AM UTC. 2024-04-26T04:46:01.862 INFO:teuthology.orchestra.run.smithi116.stdout:Dependencies resolved. 2024-04-26T04:46:01.864 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:46:01.864 INFO:teuthology.orchestra.run.smithi116.stdout: Package Arch Version Repository Size 2024-04-26T04:46:01.864 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:46:01.864 INFO:teuthology.orchestra.run.smithi116.stdout:Installing: 2024-04-26T04:46:01.865 INFO:teuthology.orchestra.run.smithi116.stdout: ceph x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 6.5 k 2024-04-26T04:46:01.865 INFO:teuthology.orchestra.run.smithi116.stdout:Installing dependencies: 2024-04-26T04:46:01.865 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-mds x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 2.1 M 2024-04-26T04:46:01.865 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-mgr x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 1.4 M 2024-04-26T04:46:01.865 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-mgr-modules-core noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 240 k 2024-04-26T04:46:01.865 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-mon x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 3.9 M 2024-04-26T04:46:01.865 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-osd x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 15 M 2024-04-26T04:46:01.865 INFO:teuthology.orchestra.run.smithi116.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-04-26T04:46:01.865 INFO:teuthology.orchestra.run.smithi116.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-04-26T04:46:01.865 INFO:teuthology.orchestra.run.smithi116.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-04-26T04:46:01.865 INFO:teuthology.orchestra.run.smithi116.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-04-26T04:46:01.865 INFO:teuthology.orchestra.run.smithi116.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-04-26T04:46:01.865 INFO:teuthology.orchestra.run.smithi116.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-04-26T04:46:01.866 INFO:teuthology.orchestra.run.smithi116.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-04-26T04:46:01.866 INFO:teuthology.orchestra.run.smithi116.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-04-26T04:46:01.866 INFO:teuthology.orchestra.run.smithi116.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-04-26T04:46:01.866 INFO:teuthology.orchestra.run.smithi116.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-04-26T04:46:01.866 INFO:teuthology.orchestra.run.smithi116.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-04-26T04:46:01.866 INFO:teuthology.orchestra.run.smithi116.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-04-26T04:46:01.866 INFO:teuthology.orchestra.run.smithi116.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-04-26T04:46:01.866 INFO:teuthology.orchestra.run.smithi116.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-04-26T04:46:01.866 INFO:teuthology.orchestra.run.smithi116.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-04-26T04:46:01.866 INFO:teuthology.orchestra.run.smithi116.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-04-26T04:46:01.866 INFO:teuthology.orchestra.run.smithi116.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-04-26T04:46:01.866 INFO:teuthology.orchestra.run.smithi116.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-04-26T04:46:01.866 INFO:teuthology.orchestra.run.smithi116.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-04-26T04:46:01.867 INFO:teuthology.orchestra.run.smithi116.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-04-26T04:46:01.867 INFO:teuthology.orchestra.run.smithi116.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-04-26T04:46:01.867 INFO:teuthology.orchestra.run.smithi116.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-04-26T04:46:01.867 INFO:teuthology.orchestra.run.smithi116.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-04-26T04:46:01.867 INFO:teuthology.orchestra.run.smithi116.stdout: python3-urllib3 noarch 1.26.5-5.el9 baseos 215 k 2024-04-26T04:46:01.867 INFO:teuthology.orchestra.run.smithi116.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-04-26T04:46:01.867 INFO:teuthology.orchestra.run.smithi116.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-04-26T04:46:01.867 INFO:teuthology.orchestra.run.smithi116.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-04-26T04:46:01.867 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:01.867 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction Summary 2024-04-26T04:46:01.867 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:46:01.867 INFO:teuthology.orchestra.run.smithi116.stdout:Install 33 Packages 2024-04-26T04:46:01.867 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:01.868 INFO:teuthology.orchestra.run.smithi116.stdout:Total download size: 27 M 2024-04-26T04:46:01.868 INFO:teuthology.orchestra.run.smithi116.stdout:Installed size: 96 M 2024-04-26T04:46:01.868 INFO:teuthology.orchestra.run.smithi116.stdout:Downloading Packages: 2024-04-26T04:46:02.657 INFO:teuthology.orchestra.run.smithi116.stdout:(1/33): ceph-17.2.7-904.gd8c6b0a3.el9.x86_64.rp 27 kB/s | 6.5 kB 00:00 2024-04-26T04:46:02.790 INFO:teuthology.orchestra.run.smithi116.stdout:(2/33): ceph-mds-17.2.7-904.gd8c6b0a3.el9.x86_6 5.6 MB/s | 2.1 MB 00:00 2024-04-26T04:46:03.008 INFO:teuthology.orchestra.run.smithi116.stdout:(3/33): ceph-mgr-17.2.7-904.gd8c6b0a3.el9.x86_6 2.4 MB/s | 1.4 MB 00:00 2024-04-26T04:46:03.116 INFO:teuthology.orchestra.run.smithi116.stdout:(4/33): ceph-mgr-modules-core-17.2.7-904.gd8c6b 2.2 MB/s | 240 kB 00:00 2024-04-26T04:46:03.575 INFO:teuthology.orchestra.run.smithi116.stdout:(5/33): ceph-osd-17.2.7-904.gd8c6b0a3.el9.x86_6 20 MB/s | 15 MB 00:00 2024-04-26T04:46:03.895 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/4 2024-04-26T04:46:03.969 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/4 2024-04-26T04:46:04.000 INFO:teuthology.orchestra.run.smithi116.stdout:(6/33): ceph-mon-17.2.7-904.gd8c6b0a3.el9.x86_6 2.9 MB/s | 3.9 MB 00:01 2024-04-26T04:46:04.334 INFO:teuthology.orchestra.run.smithi116.stdout:(7/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 208 kB/s | 253 kB 00:01 2024-04-26T04:46:04.718 INFO:teuthology.orchestra.run.smithi116.stdout:(8/33): python3-ply-3.11-14.el9.noarch.rpm 148 kB/s | 106 kB 00:00 2024-04-26T04:46:04.751 INFO:teuthology.orchestra.run.smithi116.stdout:(9/33): python3-cryptography-36.0.1-4.el9.x86_6 1.0 MB/s | 1.2 MB 00:01 2024-04-26T04:46:04.776 INFO:teuthology.orchestra.run.smithi116.stdout:(10/33): python3-pycparser-2.20-6.el9.noarch.rp 306 kB/s | 135 kB 00:00 2024-04-26T04:46:04.910 INFO:teuthology.orchestra.run.smithi116.stdout:(11/33): python3-requests-2.25.1-8.el9.noarch.r 651 kB/s | 125 kB 00:00 2024-04-26T04:46:04.944 INFO:teuthology.orchestra.run.smithi116.stdout:(12/33): python3-urllib3-1.26.5-5.el9.noarch.rp 1.1 MB/s | 215 kB 00:00 2024-04-26T04:46:04.969 INFO:teuthology.orchestra.run.smithi116.stdout:(13/33): python3-mako-1.1.4-6.el9.noarch.rpm 895 kB/s | 172 kB 00:00 2024-04-26T04:46:05.077 INFO:teuthology.orchestra.run.smithi116.stdout:(14/33): python3-markupsafe-1.1.1-12.el9.x86_64 208 kB/s | 35 kB 00:00 2024-04-26T04:46:05.111 INFO:teuthology.orchestra.run.smithi116.stdout:(15/33): python3-pytz-2021.1-5.el9.noarch.rpm 305 kB/s | 51 kB 00:00 2024-04-26T04:46:05.145 INFO:teuthology.orchestra.run.smithi116.stdout:(16/33): python3-toml-0.10.2-6.el9.noarch.rpm 237 kB/s | 42 kB 00:00 2024-04-26T04:46:05.437 INFO:teuthology.orchestra.run.smithi116.stdout:(17/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 121 kB/s | 43 kB 00:00 2024-04-26T04:46:05.628 INFO:teuthology.orchestra.run.smithi116.stdout:(18/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 56 kB/s | 11 kB 00:00 2024-04-26T04:46:05.662 INFO:teuthology.orchestra.run.smithi116.stdout:(19/33): python3-cherrypy-18.6.1-2.el9.noarch.r 693 kB/s | 358 kB 00:00 2024-04-26T04:46:05.696 INFO:teuthology.orchestra.run.smithi116.stdout:(20/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 294 kB/s | 172 kB 00:00 2024-04-26T04:46:05.904 INFO:teuthology.orchestra.run.smithi116.stdout:(21/33): python3-jaraco-classes-3.2.1-5.el9.noa 64 kB/s | 18 kB 00:00 2024-04-26T04:46:05.930 INFO:teuthology.orchestra.run.smithi116.stdout:(22/33): python3-jaraco-collections-3.0.0-8.el9 87 kB/s | 23 kB 00:00 2024-04-26T04:46:05.972 INFO:teuthology.orchestra.run.smithi116.stdout:(23/33): python3-jaraco-functools-3.5.0-2.el9.n 70 kB/s | 19 kB 00:00 2024-04-26T04:46:06.178 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/4 2024-04-26T04:46:06.178 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/4 2024-04-26T04:46:06.179 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/4 2024-04-26T04:46:06.179 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 3/4 2024-04-26T04:46:06.280 INFO:teuthology.orchestra.run.smithi116.stdout:(24/33): python3-jaraco-text-3.2.0-6.el9.noarch 52 kB/s | 20 kB 00:00 2024-04-26T04:46:06.306 INFO:teuthology.orchestra.run.smithi116.stdout:(25/33): python3-logutils-0.3.5-21.el9.noarch.r 123 kB/s | 46 kB 00:00 2024-04-26T04:46:06.331 INFO:teuthology.orchestra.run.smithi116.stdout:(26/33): python3-more-itertools-8.12.0-2.el9.no 219 kB/s | 79 kB 00:00 2024-04-26T04:46:06.481 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-04-26T04:46:06.481 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:06.481 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-04-26T04:46:06.481 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:06.481 INFO:teuthology.orchestra.run.smithi012.stdout: libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:06.481 INFO:teuthology.orchestra.run.smithi012.stdout: socat-1.7.4.1-5.el9.x86_64 2024-04-26T04:46:06.481 INFO:teuthology.orchestra.run.smithi012.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-04-26T04:46:06.481 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:06.481 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-04-26T04:46:06.581 INFO:teuthology.orchestra.run.smithi116.stdout:(27/33): python3-portend-3.1.0-2.el9.noarch.rpm 60 kB/s | 16 kB 00:00 2024-04-26T04:46:06.756 INFO:teuthology.orchestra.run.smithi116.stdout:(28/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 211 kB/s | 90 kB 00:00 2024-04-26T04:46:06.771 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph 2024-04-26T04:46:06.781 INFO:teuthology.orchestra.run.smithi116.stdout:(29/33): python3-pecan-1.4.2-3.el9.noarch.rpm 543 kB/s | 272 kB 00:00 2024-04-26T04:46:06.849 INFO:teuthology.orchestra.run.smithi116.stdout:(30/33): python3-tempora-5.0.0-2.el9.noarch.rpm 134 kB/s | 36 kB 00:00 2024-04-26T04:46:07.142 INFO:teuthology.orchestra.run.smithi116.stdout:(31/33): python3-zc-lockfile-2.0-10.el9.noarch. 68 kB/s | 20 kB 00:00 2024-04-26T04:46:07.266 INFO:teuthology.orchestra.run.smithi116.stdout:(32/33): python3-webob-1.8.7-6.el9.noarch.rpm 451 kB/s | 230 kB 00:00 2024-04-26T04:46:07.300 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:00:44 ago on Fri 26 Apr 2024 04:45:23 AM UTC. 2024-04-26T04:46:07.374 INFO:teuthology.orchestra.run.smithi116.stdout:(33/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 721 kB/s | 427 kB 00:00 2024-04-26T04:46:07.379 INFO:teuthology.orchestra.run.smithi116.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:46:07.379 INFO:teuthology.orchestra.run.smithi116.stdout:Total 4.9 MB/s | 27 MB 00:05 2024-04-26T04:46:07.427 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-04-26T04:46:07.430 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:46:07.430 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2024-04-26T04:46:07.430 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:46:07.430 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-04-26T04:46:07.430 INFO:teuthology.orchestra.run.smithi012.stdout: ceph x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 6.5 k 2024-04-26T04:46:07.430 INFO:teuthology.orchestra.run.smithi012.stdout:Installing dependencies: 2024-04-26T04:46:07.430 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mds x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 2.1 M 2024-04-26T04:46:07.430 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 1.4 M 2024-04-26T04:46:07.430 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-modules-core noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 240 k 2024-04-26T04:46:07.431 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mon x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 3.9 M 2024-04-26T04:46:07.431 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-osd x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 15 M 2024-04-26T04:46:07.431 INFO:teuthology.orchestra.run.smithi012.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-04-26T04:46:07.431 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-04-26T04:46:07.431 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-04-26T04:46:07.431 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-04-26T04:46:07.431 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-04-26T04:46:07.431 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-04-26T04:46:07.431 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-04-26T04:46:07.431 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-04-26T04:46:07.431 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-04-26T04:46:07.431 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-04-26T04:46:07.431 INFO:teuthology.orchestra.run.smithi012.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-04-26T04:46:07.432 INFO:teuthology.orchestra.run.smithi012.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-04-26T04:46:07.432 INFO:teuthology.orchestra.run.smithi012.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-04-26T04:46:07.432 INFO:teuthology.orchestra.run.smithi012.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-04-26T04:46:07.432 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-04-26T04:46:07.432 INFO:teuthology.orchestra.run.smithi012.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-04-26T04:46:07.432 INFO:teuthology.orchestra.run.smithi012.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-04-26T04:46:07.432 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-04-26T04:46:07.432 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-04-26T04:46:07.432 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-04-26T04:46:07.432 INFO:teuthology.orchestra.run.smithi012.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-04-26T04:46:07.432 INFO:teuthology.orchestra.run.smithi012.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-04-26T04:46:07.432 INFO:teuthology.orchestra.run.smithi012.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-04-26T04:46:07.432 INFO:teuthology.orchestra.run.smithi012.stdout: python3-urllib3 noarch 1.26.5-5.el9 baseos 215 k 2024-04-26T04:46:07.432 INFO:teuthology.orchestra.run.smithi012.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-04-26T04:46:07.433 INFO:teuthology.orchestra.run.smithi012.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-04-26T04:46:07.433 INFO:teuthology.orchestra.run.smithi012.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-04-26T04:46:07.433 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:07.433 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-04-26T04:46:07.433 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:46:07.433 INFO:teuthology.orchestra.run.smithi012.stdout:Install 33 Packages 2024-04-26T04:46:07.433 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:07.434 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 27 M 2024-04-26T04:46:07.434 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 96 M 2024-04-26T04:46:07.434 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-04-26T04:46:07.498 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction check 2024-04-26T04:46:07.535 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction check succeeded. 2024-04-26T04:46:07.535 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction test 2024-04-26T04:46:07.801 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction test succeeded. 2024-04-26T04:46:07.802 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction 2024-04-26T04:46:08.202 INFO:teuthology.orchestra.run.smithi012.stdout:(1/33): ceph-17.2.7-904.gd8c6b0a3.el9.x86_64.rp 39 kB/s | 6.5 kB 00:00 2024-04-26T04:46:08.478 INFO:teuthology.orchestra.run.smithi012.stdout:(2/33): ceph-mgr-17.2.7-904.gd8c6b0a3.el9.x86_6 3.2 MB/s | 1.4 MB 00:00 2024-04-26T04:46:08.520 INFO:teuthology.orchestra.run.smithi012.stdout:(3/33): ceph-mds-17.2.7-904.gd8c6b0a3.el9.x86_6 4.3 MB/s | 2.1 MB 00:00 2024-04-26T04:46:08.586 INFO:teuthology.orchestra.run.smithi012.stdout:(4/33): ceph-mgr-modules-core-17.2.7-904.gd8c6b 3.5 MB/s | 240 kB 00:00 2024-04-26T04:46:08.737 INFO:teuthology.orchestra.run.smithi012.stdout:(5/33): ceph-mon-17.2.7-904.gd8c6b0a3.el9.x86_6 7.3 MB/s | 3.9 MB 00:00 2024-04-26T04:46:08.762 INFO:teuthology.orchestra.run.smithi012.stdout:(6/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.4 MB/s | 253 kB 00:00 2024-04-26T04:46:08.821 INFO:teuthology.orchestra.run.smithi012.stdout:(7/33): python3-cryptography-36.0.1-4.el9.x86_6 15 MB/s | 1.2 MB 00:00 2024-04-26T04:46:08.846 INFO:teuthology.orchestra.run.smithi012.stdout:(8/33): python3-ply-3.11-14.el9.noarch.rpm 1.3 MB/s | 106 kB 00:00 2024-04-26T04:46:08.871 INFO:teuthology.orchestra.run.smithi012.stdout:(9/33): python3-pycparser-2.20-6.el9.noarch.rpm 2.6 MB/s | 135 kB 00:00 2024-04-26T04:46:08.897 INFO:teuthology.orchestra.run.smithi012.stdout:(10/33): python3-requests-2.25.1-8.el9.noarch.r 2.4 MB/s | 125 kB 00:00 2024-04-26T04:46:08.922 INFO:teuthology.orchestra.run.smithi012.stdout:(11/33): python3-urllib3-1.26.5-5.el9.noarch.rp 4.2 MB/s | 215 kB 00:00 2024-04-26T04:46:08.925 INFO:teuthology.orchestra.run.smithi116.stdout: Preparing : 1/1 2024-04-26T04:46:08.947 INFO:teuthology.orchestra.run.smithi012.stdout:(12/33): python3-mako-1.1.4-6.el9.noarch.rpm 3.4 MB/s | 172 kB 00:00 2024-04-26T04:46:08.964 INFO:teuthology.orchestra.run.smithi012.stdout:(13/33): python3-markupsafe-1.1.1-12.el9.x86_64 831 kB/s | 35 kB 00:00 2024-04-26T04:46:08.981 INFO:teuthology.orchestra.run.smithi012.stdout:(14/33): python3-pytz-2021.1-5.el9.noarch.rpm 1.5 MB/s | 51 kB 00:00 2024-04-26T04:46:08.994 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-04-26T04:46:08.998 INFO:teuthology.orchestra.run.smithi012.stdout:(15/33): python3-toml-0.10.2-6.el9.noarch.rpm 1.2 MB/s | 42 kB 00:00 2024-04-26T04:46:09.026 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-04-26T04:46:09.059 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-04-26T04:46:09.159 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-04-26T04:46:09.254 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-04-26T04:46:09.295 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 6/33 2024-04-26T04:46:09.330 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-04-26T04:46:09.360 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/33 2024-04-26T04:46:09.394 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/33 2024-04-26T04:46:09.425 INFO:teuthology.orchestra.run.smithi012.stdout:(16/33): ceph-osd-17.2.7-904.gd8c6b0a3.el9.x86_ 16 MB/s | 15 MB 00:00 2024-04-26T04:46:09.428 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/33 2024-04-26T04:46:09.440 INFO:teuthology.orchestra.run.smithi012.stdout:(17/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 95 kB/s | 43 kB 00:00 2024-04-26T04:46:09.468 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/33 2024-04-26T04:46:09.512 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/33 2024-04-26T04:46:09.546 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/33 2024-04-26T04:46:09.584 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/33 2024-04-26T04:46:09.615 INFO:teuthology.orchestra.run.smithi012.stdout:(18/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 278 kB/s | 172 kB 00:00 2024-04-26T04:46:09.632 INFO:teuthology.orchestra.run.smithi012.stdout:(19/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 56 kB/s | 11 kB 00:00 2024-04-26T04:46:09.665 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 15/33 2024-04-26T04:46:09.707 INFO:teuthology.orchestra.run.smithi012.stdout:(20/33): python3-jaraco-classes-3.2.1-5.el9.noa 193 kB/s | 18 kB 00:00 2024-04-26T04:46:09.724 INFO:teuthology.orchestra.run.smithi012.stdout:(21/33): python3-jaraco-collections-3.0.0-8.el9 251 kB/s | 23 kB 00:00 2024-04-26T04:46:09.800 INFO:teuthology.orchestra.run.smithi012.stdout:(22/33): python3-jaraco-functools-3.5.0-2.el9.n 211 kB/s | 19 kB 00:00 2024-04-26T04:46:09.812 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 16/33 2024-04-26T04:46:09.817 INFO:teuthology.orchestra.run.smithi012.stdout:(23/33): python3-jaraco-text-3.2.0-6.el9.noarch 214 kB/s | 20 kB 00:00 2024-04-26T04:46:09.850 INFO:teuthology.orchestra.run.smithi012.stdout:(24/33): python3-cherrypy-18.6.1-2.el9.noarch.r 840 kB/s | 358 kB 00:00 2024-04-26T04:46:09.892 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 17/33 2024-04-26T04:46:09.892 INFO:teuthology.orchestra.run.smithi012.stdout:(25/33): python3-logutils-0.3.5-21.el9.noarch.r 502 kB/s | 46 kB 00:00 2024-04-26T04:46:09.945 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 18/33 2024-04-26T04:46:09.983 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 19/33 2024-04-26T04:46:09.984 INFO:teuthology.orchestra.run.smithi012.stdout:(26/33): python3-more-itertools-8.12.0-2.el9.no 472 kB/s | 79 kB 00:00 2024-04-26T04:46:10.009 INFO:teuthology.orchestra.run.smithi012.stdout:(27/33): python3-portend-3.1.0-2.el9.noarch.rpm 140 kB/s | 16 kB 00:00 2024-04-26T04:46:10.032 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-ply-3.11-14.el9.noarch 20/33 2024-04-26T04:46:10.093 INFO:teuthology.orchestra.run.smithi012.stdout:(28/33): python3-pecan-1.4.2-3.el9.noarch.rpm 1.1 MB/s | 272 kB 00:00 2024-04-26T04:46:10.096 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 21/33 2024-04-26T04:46:10.143 INFO:teuthology.orchestra.run.smithi012.stdout:(29/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 566 kB/s | 90 kB 00:00 2024-04-26T04:46:10.169 INFO:teuthology.orchestra.run.smithi012.stdout:(30/33): python3-tempora-5.0.0-2.el9.noarch.rpm 225 kB/s | 36 kB 00:00 2024-04-26T04:46:10.260 INFO:teuthology.orchestra.run.smithi012.stdout:(31/33): python3-webob-1.8.7-6.el9.noarch.rpm 1.3 MB/s | 230 kB 00:00 2024-04-26T04:46:10.285 INFO:teuthology.orchestra.run.smithi012.stdout:(32/33): python3-zc-lockfile-2.0-10.el9.noarch. 171 kB/s | 20 kB 00:00 2024-04-26T04:46:10.291 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 22/33 2024-04-26T04:46:10.327 INFO:teuthology.orchestra.run.smithi012.stdout:(33/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 2.3 MB/s | 427 kB 00:00 2024-04-26T04:46:10.331 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:46:10.331 INFO:teuthology.orchestra.run.smithi012.stdout:Total 9.4 MB/s | 27 MB 00:02 2024-04-26T04:46:10.346 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 23/33 2024-04-26T04:46:10.416 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 24/33 2024-04-26T04:46:10.453 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-04-26T04:46:10.493 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-04-26T04:46:10.494 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-04-26T04:46:10.505 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 25/33 2024-04-26T04:46:10.545 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 26/33 2024-04-26T04:46:10.662 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 27/33 2024-04-26T04:46:10.778 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-04-26T04:46:10.778 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-04-26T04:46:10.846 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-904.gd8c6b0a3.el9.n 28/33 2024-04-26T04:46:10.881 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 29/33 2024-04-26T04:46:10.906 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 29/33 2024-04-26T04:46:10.906 INFO:teuthology.orchestra.run.smithi116.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:46:10.906 INFO:teuthology.orchestra.run.smithi116.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-04-26T04:46:10.906 INFO:teuthology.orchestra.run.smithi116.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-26T04:46:10.907 INFO:teuthology.orchestra.run.smithi116.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-26T04:46:10.907 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:11.961 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-04-26T04:46:12.039 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-04-26T04:46:12.072 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-04-26T04:46:12.104 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-04-26T04:46:12.200 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-04-26T04:46:12.215 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 30/33 2024-04-26T04:46:12.241 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 30/33 2024-04-26T04:46:12.242 INFO:teuthology.orchestra.run.smithi116.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:46:12.242 INFO:teuthology.orchestra.run.smithi116.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-04-26T04:46:12.242 INFO:teuthology.orchestra.run.smithi116.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-26T04:46:12.242 INFO:teuthology.orchestra.run.smithi116.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-26T04:46:12.242 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:12.278 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-04-26T04:46:12.333 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 6/33 2024-04-26T04:46:12.367 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-04-26T04:46:12.396 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/33 2024-04-26T04:46:12.441 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/33 2024-04-26T04:46:12.473 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/33 2024-04-26T04:46:12.514 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/33 2024-04-26T04:46:12.557 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/33 2024-04-26T04:46:12.591 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/33 2024-04-26T04:46:12.616 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 31/33 2024-04-26T04:46:12.638 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/33 2024-04-26T04:46:12.639 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 31/33 2024-04-26T04:46:12.639 INFO:teuthology.orchestra.run.smithi116.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:46:12.640 INFO:teuthology.orchestra.run.smithi116.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-04-26T04:46:12.640 INFO:teuthology.orchestra.run.smithi116.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-26T04:46:12.640 INFO:teuthology.orchestra.run.smithi116.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-26T04:46:12.640 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:12.715 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 15/33 2024-04-26T04:46:12.841 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 32/33 2024-04-26T04:46:12.864 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 32/33 2024-04-26T04:46:12.864 INFO:teuthology.orchestra.run.smithi116.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:46:12.864 INFO:teuthology.orchestra.run.smithi116.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-04-26T04:46:12.864 INFO:teuthology.orchestra.run.smithi116.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-26T04:46:12.864 INFO:teuthology.orchestra.run.smithi116.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-26T04:46:12.864 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:12.875 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 16/33 2024-04-26T04:46:12.928 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 33/33 2024-04-26T04:46:12.957 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 17/33 2024-04-26T04:46:13.016 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 18/33 2024-04-26T04:46:13.054 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 19/33 2024-04-26T04:46:13.113 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-ply-3.11-14.el9.noarch 20/33 2024-04-26T04:46:13.177 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 21/33 2024-04-26T04:46:13.373 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 22/33 2024-04-26T04:46:13.425 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 23/33 2024-04-26T04:46:13.507 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 24/33 2024-04-26T04:46:13.604 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 25/33 2024-04-26T04:46:13.641 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 26/33 2024-04-26T04:46:13.764 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 27/33 2024-04-26T04:46:13.918 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-904.gd8c6b0a3.el9.n 28/33 2024-04-26T04:46:13.942 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 29/33 2024-04-26T04:46:13.968 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 29/33 2024-04-26T04:46:13.968 INFO:teuthology.orchestra.run.smithi012.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:46:13.968 INFO:teuthology.orchestra.run.smithi012.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-04-26T04:46:13.968 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-26T04:46:13.968 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-26T04:46:13.968 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:14.443 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 33/33 2024-04-26T04:46:14.443 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/33 2024-04-26T04:46:14.443 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/33 2024-04-26T04:46:14.443 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/33 2024-04-26T04:46:14.443 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/33 2024-04-26T04:46:14.443 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 5/33 2024-04-26T04:46:14.443 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-904.gd8c6b0a3.el9.n 6/33 2024-04-26T04:46:14.443 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/33 2024-04-26T04:46:14.444 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/33 2024-04-26T04:46:14.444 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/33 2024-04-26T04:46:14.444 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/33 2024-04-26T04:46:14.444 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/33 2024-04-26T04:46:14.444 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 12/33 2024-04-26T04:46:14.444 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/33 2024-04-26T04:46:14.444 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/33 2024-04-26T04:46:14.444 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 15/33 2024-04-26T04:46:14.444 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 16/33 2024-04-26T04:46:14.444 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-04-26T04:46:14.444 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-04-26T04:46:14.444 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-04-26T04:46:14.444 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-04-26T04:46:14.444 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-04-26T04:46:14.445 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-04-26T04:46:14.445 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-04-26T04:46:14.445 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-04-26T04:46:14.445 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-04-26T04:46:14.445 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-04-26T04:46:14.445 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-04-26T04:46:14.445 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-04-26T04:46:14.445 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-04-26T04:46:14.445 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-04-26T04:46:14.445 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 31/33 2024-04-26T04:46:14.445 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-04-26T04:46:15.365 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-04-26T04:46:15.381 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:15.382 INFO:teuthology.orchestra.run.smithi116.stdout:Installed: 2024-04-26T04:46:15.382 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:15.382 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:15.382 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:15.382 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-mgr-modules-core-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:46:15.382 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:15.382 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:15.382 INFO:teuthology.orchestra.run.smithi116.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-04-26T04:46:15.382 INFO:teuthology.orchestra.run.smithi116.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-04-26T04:46:15.382 INFO:teuthology.orchestra.run.smithi116.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-04-26T04:46:15.382 INFO:teuthology.orchestra.run.smithi116.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-04-26T04:46:15.383 INFO:teuthology.orchestra.run.smithi116.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-04-26T04:46:15.383 INFO:teuthology.orchestra.run.smithi116.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-04-26T04:46:15.383 INFO:teuthology.orchestra.run.smithi116.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-04-26T04:46:15.383 INFO:teuthology.orchestra.run.smithi116.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-04-26T04:46:15.383 INFO:teuthology.orchestra.run.smithi116.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-04-26T04:46:15.383 INFO:teuthology.orchestra.run.smithi116.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-04-26T04:46:15.383 INFO:teuthology.orchestra.run.smithi116.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-04-26T04:46:15.383 INFO:teuthology.orchestra.run.smithi116.stdout: python3-mako-1.1.4-6.el9.noarch 2024-04-26T04:46:15.383 INFO:teuthology.orchestra.run.smithi116.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-04-26T04:46:15.383 INFO:teuthology.orchestra.run.smithi116.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-04-26T04:46:15.383 INFO:teuthology.orchestra.run.smithi116.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-04-26T04:46:15.383 INFO:teuthology.orchestra.run.smithi116.stdout: python3-ply-3.11-14.el9.noarch 2024-04-26T04:46:15.383 INFO:teuthology.orchestra.run.smithi116.stdout: python3-portend-3.1.0-2.el9.noarch 2024-04-26T04:46:15.384 INFO:teuthology.orchestra.run.smithi116.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-04-26T04:46:15.384 INFO:teuthology.orchestra.run.smithi116.stdout: python3-pycparser-2.20-6.el9.noarch 2024-04-26T04:46:15.384 INFO:teuthology.orchestra.run.smithi116.stdout: python3-pytz-2021.1-5.el9.noarch 2024-04-26T04:46:15.384 INFO:teuthology.orchestra.run.smithi116.stdout: python3-requests-2.25.1-8.el9.noarch 2024-04-26T04:46:15.384 INFO:teuthology.orchestra.run.smithi116.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-04-26T04:46:15.384 INFO:teuthology.orchestra.run.smithi116.stdout: python3-toml-0.10.2-6.el9.noarch 2024-04-26T04:46:15.384 INFO:teuthology.orchestra.run.smithi116.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-04-26T04:46:15.384 INFO:teuthology.orchestra.run.smithi116.stdout: python3-webob-1.8.7-6.el9.noarch 2024-04-26T04:46:15.384 INFO:teuthology.orchestra.run.smithi116.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-04-26T04:46:15.384 INFO:teuthology.orchestra.run.smithi116.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-04-26T04:46:15.384 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:15.384 INFO:teuthology.orchestra.run.smithi116.stdout:Complete! 2024-04-26T04:46:15.385 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 30/33 2024-04-26T04:46:15.398 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 30/33 2024-04-26T04:46:15.398 INFO:teuthology.orchestra.run.smithi012.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:46:15.398 INFO:teuthology.orchestra.run.smithi012.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-04-26T04:46:15.398 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-26T04:46:15.398 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-26T04:46:15.398 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:15.745 DEBUG:teuthology.orchestra.run.smithi116:> sudo yum -y install ceph-base 2024-04-26T04:46:15.786 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 31/33 2024-04-26T04:46:15.812 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 31/33 2024-04-26T04:46:15.812 INFO:teuthology.orchestra.run.smithi012.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:46:15.812 INFO:teuthology.orchestra.run.smithi012.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-04-26T04:46:15.812 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-26T04:46:15.812 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-26T04:46:15.812 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:16.020 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 32/33 2024-04-26T04:46:16.045 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 32/33 2024-04-26T04:46:16.045 INFO:teuthology.orchestra.run.smithi012.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:46:16.045 INFO:teuthology.orchestra.run.smithi012.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-04-26T04:46:16.045 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-26T04:46:16.045 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-26T04:46:16.046 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:16.119 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 33/33 2024-04-26T04:46:16.255 INFO:teuthology.orchestra.run.smithi116.stdout:Last metadata expiration check: 0:00:54 ago on Fri 26 Apr 2024 04:45:22 AM UTC. 2024-04-26T04:46:16.333 INFO:teuthology.orchestra.run.smithi116.stdout:Package ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:46:16.373 INFO:teuthology.orchestra.run.smithi116.stdout:Dependencies resolved. 2024-04-26T04:46:16.374 INFO:teuthology.orchestra.run.smithi116.stdout:Nothing to do. 2024-04-26T04:46:16.374 INFO:teuthology.orchestra.run.smithi116.stdout:Complete! 2024-04-26T04:46:16.450 DEBUG:teuthology.orchestra.run.smithi116:> sudo yum -y install cephadm 2024-04-26T04:46:16.955 INFO:teuthology.orchestra.run.smithi116.stdout:Last metadata expiration check: 0:00:54 ago on Fri 26 Apr 2024 04:45:22 AM UTC. 2024-04-26T04:46:17.075 INFO:teuthology.orchestra.run.smithi116.stdout:Dependencies resolved. 2024-04-26T04:46:17.075 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:46:17.075 INFO:teuthology.orchestra.run.smithi116.stdout: Package Arch Version Repository Size 2024-04-26T04:46:17.075 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:46:17.075 INFO:teuthology.orchestra.run.smithi116.stdout:Installing: 2024-04-26T04:46:17.075 INFO:teuthology.orchestra.run.smithi116.stdout: cephadm noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 82 k 2024-04-26T04:46:17.075 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:17.076 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction Summary 2024-04-26T04:46:17.076 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:46:17.076 INFO:teuthology.orchestra.run.smithi116.stdout:Install 1 Package 2024-04-26T04:46:17.076 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:17.076 INFO:teuthology.orchestra.run.smithi116.stdout:Total download size: 82 k 2024-04-26T04:46:17.076 INFO:teuthology.orchestra.run.smithi116.stdout:Installed size: 372 k 2024-04-26T04:46:17.076 INFO:teuthology.orchestra.run.smithi116.stdout:Downloading Packages: 2024-04-26T04:46:17.390 INFO:teuthology.orchestra.run.smithi116.stdout:cephadm-17.2.7-904.gd8c6b0a3.el9.noarch.rpm 261 kB/s | 82 kB 00:00 2024-04-26T04:46:17.391 INFO:teuthology.orchestra.run.smithi116.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:46:17.391 INFO:teuthology.orchestra.run.smithi116.stdout:Total 260 kB/s | 82 kB 00:00 2024-04-26T04:46:17.391 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction check 2024-04-26T04:46:17.396 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction check succeeded. 2024-04-26T04:46:17.396 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction test 2024-04-26T04:46:17.398 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction test succeeded. 2024-04-26T04:46:17.398 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction 2024-04-26T04:46:17.473 INFO:teuthology.orchestra.run.smithi116.stdout: Preparing : 1/1 2024-04-26T04:46:17.937 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 33/33 2024-04-26T04:46:17.937 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/33 2024-04-26T04:46:17.937 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/33 2024-04-26T04:46:17.937 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/33 2024-04-26T04:46:17.937 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/33 2024-04-26T04:46:17.938 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 5/33 2024-04-26T04:46:17.938 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-904.gd8c6b0a3.el9.n 6/33 2024-04-26T04:46:17.938 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/33 2024-04-26T04:46:17.938 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/33 2024-04-26T04:46:17.938 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/33 2024-04-26T04:46:17.938 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/33 2024-04-26T04:46:17.938 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/33 2024-04-26T04:46:17.938 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 12/33 2024-04-26T04:46:17.938 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/33 2024-04-26T04:46:17.938 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/33 2024-04-26T04:46:17.938 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 15/33 2024-04-26T04:46:17.938 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 16/33 2024-04-26T04:46:17.938 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-04-26T04:46:17.939 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-04-26T04:46:17.939 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-04-26T04:46:17.939 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-04-26T04:46:17.939 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-04-26T04:46:17.939 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-04-26T04:46:17.939 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-04-26T04:46:17.939 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-04-26T04:46:17.939 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-04-26T04:46:17.939 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-04-26T04:46:17.939 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-04-26T04:46:17.940 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-04-26T04:46:17.940 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-04-26T04:46:17.940 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-04-26T04:46:17.940 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 31/33 2024-04-26T04:46:17.940 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-04-26T04:46:17.954 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:46:18.024 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:46:18.333 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:46:18.525 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:46:18.525 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:18.525 INFO:teuthology.orchestra.run.smithi116.stdout:Installed: 2024-04-26T04:46:18.525 INFO:teuthology.orchestra.run.smithi116.stdout: cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:46:18.525 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:18.525 INFO:teuthology.orchestra.run.smithi116.stdout:Complete! 2024-04-26T04:46:18.709 DEBUG:teuthology.orchestra.run.smithi116:> sudo yum -y install ceph-immutable-object-cache 2024-04-26T04:46:18.789 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-04-26T04:46:18.789 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:18.789 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-04-26T04:46:18.789 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:18.789 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:18.789 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:18.789 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-modules-core-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:46:18.789 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:18.789 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:18.789 INFO:teuthology.orchestra.run.smithi012.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-04-26T04:46:18.789 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-04-26T04:46:18.790 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-04-26T04:46:18.790 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-04-26T04:46:18.790 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-04-26T04:46:18.790 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-04-26T04:46:18.790 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-04-26T04:46:18.790 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-04-26T04:46:18.790 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-04-26T04:46:18.790 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-04-26T04:46:18.790 INFO:teuthology.orchestra.run.smithi012.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-04-26T04:46:18.790 INFO:teuthology.orchestra.run.smithi012.stdout: python3-mako-1.1.4-6.el9.noarch 2024-04-26T04:46:18.790 INFO:teuthology.orchestra.run.smithi012.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-04-26T04:46:18.790 INFO:teuthology.orchestra.run.smithi012.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-04-26T04:46:18.790 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-04-26T04:46:18.791 INFO:teuthology.orchestra.run.smithi012.stdout: python3-ply-3.11-14.el9.noarch 2024-04-26T04:46:18.791 INFO:teuthology.orchestra.run.smithi012.stdout: python3-portend-3.1.0-2.el9.noarch 2024-04-26T04:46:18.791 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-04-26T04:46:18.791 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pycparser-2.20-6.el9.noarch 2024-04-26T04:46:18.791 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pytz-2021.1-5.el9.noarch 2024-04-26T04:46:18.791 INFO:teuthology.orchestra.run.smithi012.stdout: python3-requests-2.25.1-8.el9.noarch 2024-04-26T04:46:18.791 INFO:teuthology.orchestra.run.smithi012.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-04-26T04:46:18.791 INFO:teuthology.orchestra.run.smithi012.stdout: python3-toml-0.10.2-6.el9.noarch 2024-04-26T04:46:18.791 INFO:teuthology.orchestra.run.smithi012.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-04-26T04:46:18.791 INFO:teuthology.orchestra.run.smithi012.stdout: python3-webob-1.8.7-6.el9.noarch 2024-04-26T04:46:18.791 INFO:teuthology.orchestra.run.smithi012.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-04-26T04:46:18.791 INFO:teuthology.orchestra.run.smithi012.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-04-26T04:46:18.791 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:18.791 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-04-26T04:46:19.121 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-base 2024-04-26T04:46:19.220 INFO:teuthology.orchestra.run.smithi116.stdout:Last metadata expiration check: 0:00:57 ago on Fri 26 Apr 2024 04:45:22 AM UTC. 2024-04-26T04:46:19.339 INFO:teuthology.orchestra.run.smithi116.stdout:Dependencies resolved. 2024-04-26T04:46:19.339 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:46:19.340 INFO:teuthology.orchestra.run.smithi116.stdout: Package Arch Version Repo Size 2024-04-26T04:46:19.340 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:46:19.340 INFO:teuthology.orchestra.run.smithi116.stdout:Installing: 2024-04-26T04:46:19.340 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 137 k 2024-04-26T04:46:19.340 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:19.340 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction Summary 2024-04-26T04:46:19.340 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:46:19.340 INFO:teuthology.orchestra.run.smithi116.stdout:Install 1 Package 2024-04-26T04:46:19.340 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:19.340 INFO:teuthology.orchestra.run.smithi116.stdout:Total download size: 137 k 2024-04-26T04:46:19.340 INFO:teuthology.orchestra.run.smithi116.stdout:Installed size: 379 k 2024-04-26T04:46:19.341 INFO:teuthology.orchestra.run.smithi116.stdout:Downloading Packages: 2024-04-26T04:46:19.555 INFO:teuthology.orchestra.run.smithi116.stdout:ceph-immutable-object-cache-17.2.7-904.gd8c6b0a 642 kB/s | 137 kB 00:00 2024-04-26T04:46:19.556 INFO:teuthology.orchestra.run.smithi116.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:46:19.556 INFO:teuthology.orchestra.run.smithi116.stdout:Total 636 kB/s | 137 kB 00:00 2024-04-26T04:46:19.556 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction check 2024-04-26T04:46:19.564 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction check succeeded. 2024-04-26T04:46:19.564 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction test 2024-04-26T04:46:19.620 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction test succeeded. 2024-04-26T04:46:19.620 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction 2024-04-26T04:46:19.653 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:00:56 ago on Fri 26 Apr 2024 04:45:23 AM UTC. 2024-04-26T04:46:19.734 INFO:teuthology.orchestra.run.smithi012.stdout:Package ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:46:19.760 INFO:teuthology.orchestra.run.smithi116.stdout: Preparing : 1/1 2024-04-26T04:46:19.776 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-04-26T04:46:19.778 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2024-04-26T04:46:19.778 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-04-26T04:46:19.822 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.e 1/1 2024-04-26T04:46:19.839 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install cephadm 2024-04-26T04:46:19.846 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.e 1/1 2024-04-26T04:46:19.846 INFO:teuthology.orchestra.run.smithi116.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:46:19.846 INFO:teuthology.orchestra.run.smithi116.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-04-26T04:46:19.846 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:20.364 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:00:57 ago on Fri 26 Apr 2024 04:45:23 AM UTC. 2024-04-26T04:46:20.487 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-04-26T04:46:20.488 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:46:20.488 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2024-04-26T04:46:20.488 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:46:20.488 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-04-26T04:46:20.488 INFO:teuthology.orchestra.run.smithi012.stdout: cephadm noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 82 k 2024-04-26T04:46:20.488 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:20.488 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-04-26T04:46:20.488 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:46:20.488 INFO:teuthology.orchestra.run.smithi012.stdout:Install 1 Package 2024-04-26T04:46:20.488 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:20.489 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 82 k 2024-04-26T04:46:20.489 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 372 k 2024-04-26T04:46:20.489 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-04-26T04:46:20.679 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.e 1/1 2024-04-26T04:46:20.679 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:20.679 INFO:teuthology.orchestra.run.smithi116.stdout:Installed: 2024-04-26T04:46:20.679 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:20.679 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:20.679 INFO:teuthology.orchestra.run.smithi116.stdout:Complete! 2024-04-26T04:46:20.803 INFO:teuthology.orchestra.run.smithi012.stdout:cephadm-17.2.7-904.gd8c6b0a3.el9.noarch.rpm 262 kB/s | 82 kB 00:00 2024-04-26T04:46:20.803 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:46:20.803 INFO:teuthology.orchestra.run.smithi012.stdout:Total 260 kB/s | 82 kB 00:00 2024-04-26T04:46:20.803 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-04-26T04:46:20.810 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-04-26T04:46:20.810 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-04-26T04:46:20.811 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-04-26T04:46:20.811 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-04-26T04:46:20.893 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-04-26T04:46:20.916 DEBUG:teuthology.orchestra.run.smithi116:> sudo yum -y install ceph-mgr 2024-04-26T04:46:21.336 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:46:21.427 INFO:teuthology.orchestra.run.smithi116.stdout:Last metadata expiration check: 0:00:59 ago on Fri 26 Apr 2024 04:45:22 AM UTC. 2024-04-26T04:46:21.432 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:46:21.506 INFO:teuthology.orchestra.run.smithi116.stdout:Package ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:46:21.548 INFO:teuthology.orchestra.run.smithi116.stdout:Dependencies resolved. 2024-04-26T04:46:21.548 INFO:teuthology.orchestra.run.smithi116.stdout:Nothing to do. 2024-04-26T04:46:21.549 INFO:teuthology.orchestra.run.smithi116.stdout:Complete! 2024-04-26T04:46:21.630 DEBUG:teuthology.orchestra.run.smithi116:> sudo yum -y install ceph-mgr-dashboard 2024-04-26T04:46:21.717 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:46:21.961 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:46:21.961 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:21.961 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-04-26T04:46:21.961 INFO:teuthology.orchestra.run.smithi012.stdout: cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:46:21.962 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:21.962 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-04-26T04:46:22.143 INFO:teuthology.orchestra.run.smithi116.stdout:Last metadata expiration check: 0:01:00 ago on Fri 26 Apr 2024 04:45:22 AM UTC. 2024-04-26T04:46:22.166 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-immutable-object-cache 2024-04-26T04:46:22.261 INFO:teuthology.orchestra.run.smithi116.stdout:Dependencies resolved. 2024-04-26T04:46:22.262 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:46:22.262 INFO:teuthology.orchestra.run.smithi116.stdout: Package Arch Version Repository Size 2024-04-26T04:46:22.262 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:46:22.262 INFO:teuthology.orchestra.run.smithi116.stdout:Installing: 2024-04-26T04:46:22.262 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-mgr-dashboard noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 3.4 M 2024-04-26T04:46:22.262 INFO:teuthology.orchestra.run.smithi116.stdout:Installing dependencies: 2024-04-26T04:46:22.262 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-grafana-dashboards noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 24 k 2024-04-26T04:46:22.262 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-prometheus-alerts noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 15 k 2024-04-26T04:46:22.262 INFO:teuthology.orchestra.run.smithi116.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-04-26T04:46:22.262 INFO:teuthology.orchestra.run.smithi116.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-04-26T04:46:22.262 INFO:teuthology.orchestra.run.smithi116.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-04-26T04:46:22.263 INFO:teuthology.orchestra.run.smithi116.stdout:Installing weak dependencies: 2024-04-26T04:46:22.263 INFO:teuthology.orchestra.run.smithi116.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-04-26T04:46:22.263 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:22.263 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction Summary 2024-04-26T04:46:22.263 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:46:22.263 INFO:teuthology.orchestra.run.smithi116.stdout:Install 7 Packages 2024-04-26T04:46:22.263 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:22.264 INFO:teuthology.orchestra.run.smithi116.stdout:Total download size: 3.7 M 2024-04-26T04:46:22.264 INFO:teuthology.orchestra.run.smithi116.stdout:Installed size: 80 M 2024-04-26T04:46:22.264 INFO:teuthology.orchestra.run.smithi116.stdout:Downloading Packages: 2024-04-26T04:46:22.575 INFO:teuthology.orchestra.run.smithi116.stdout:(1/7): ceph-prometheus-alerts-17.2.7-904.gd8c6b 63 kB/s | 15 kB 00:00 2024-04-26T04:46:22.601 INFO:teuthology.orchestra.run.smithi116.stdout:(2/7): ceph-grafana-dashboards-17.2.7-904.gd8c6 96 kB/s | 24 kB 00:00 2024-04-26T04:46:22.706 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:00:59 ago on Fri 26 Apr 2024 04:45:23 AM UTC. 2024-04-26T04:46:22.734 INFO:teuthology.orchestra.run.smithi116.stdout:(3/7): python3-jwt+crypto-2.4.0-1.el9.noarch.rp 57 kB/s | 9.0 kB 00:00 2024-04-26T04:46:22.793 INFO:teuthology.orchestra.run.smithi116.stdout:(4/7): python3-jwt-2.4.0-1.el9.noarch.rpm 213 kB/s | 41 kB 00:00 2024-04-26T04:46:22.818 INFO:teuthology.orchestra.run.smithi116.stdout:(5/7): python3-repoze-lru-0.7-16.el9.noarch.rpm 368 kB/s | 31 kB 00:00 2024-04-26T04:46:22.829 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-04-26T04:46:22.830 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:46:22.830 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repo Size 2024-04-26T04:46:22.830 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:46:22.830 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-04-26T04:46:22.830 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 137 k 2024-04-26T04:46:22.830 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:22.830 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-04-26T04:46:22.831 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:46:22.831 INFO:teuthology.orchestra.run.smithi012.stdout:Install 1 Package 2024-04-26T04:46:22.831 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:22.831 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 137 k 2024-04-26T04:46:22.831 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 379 k 2024-04-26T04:46:22.831 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-04-26T04:46:22.877 INFO:teuthology.orchestra.run.smithi116.stdout:(6/7): python3-routes-2.5.1-5.el9.noarch.rpm 2.2 MB/s | 188 kB 00:00 2024-04-26T04:46:23.058 INFO:teuthology.orchestra.run.smithi012.stdout:ceph-immutable-object-cache-17.2.7-904.gd8c6b0a 606 kB/s | 137 kB 00:00 2024-04-26T04:46:23.059 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:46:23.059 INFO:teuthology.orchestra.run.smithi012.stdout:Total 601 kB/s | 137 kB 00:00 2024-04-26T04:46:23.059 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-04-26T04:46:23.067 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-04-26T04:46:23.067 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-04-26T04:46:23.127 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-04-26T04:46:23.128 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-04-26T04:46:23.169 INFO:teuthology.orchestra.run.smithi116.stdout:(7/7): ceph-mgr-dashboard-17.2.7-904.gd8c6b0a3. 4.1 MB/s | 3.4 MB 00:00 2024-04-26T04:46:23.171 INFO:teuthology.orchestra.run.smithi116.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:46:23.172 INFO:teuthology.orchestra.run.smithi116.stdout:Total 4.0 MB/s | 3.7 MB 00:00 2024-04-26T04:46:23.172 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction check 2024-04-26T04:46:23.181 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction check succeeded. 2024-04-26T04:46:23.181 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction test 2024-04-26T04:46:23.240 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction test succeeded. 2024-04-26T04:46:23.241 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction 2024-04-26T04:46:23.310 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-04-26T04:46:23.393 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.e 1/1 2024-04-26T04:46:23.418 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.e 1/1 2024-04-26T04:46:23.419 INFO:teuthology.orchestra.run.smithi012.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:46:23.419 INFO:teuthology.orchestra.run.smithi012.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-04-26T04:46:23.419 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:23.537 INFO:teuthology.orchestra.run.smithi116.stdout: Preparing : 1/1 2024-04-26T04:46:23.612 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 1/7 2024-04-26T04:46:23.645 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 2/7 2024-04-26T04:46:23.692 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/7 2024-04-26T04:46:23.724 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/7 2024-04-26T04:46:23.753 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-904.gd8c6b0a3.el9.no 5/7 2024-04-26T04:46:24.319 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.e 1/1 2024-04-26T04:46:24.319 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:24.319 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-04-26T04:46:24.319 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:24.319 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:24.319 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-04-26T04:46:24.528 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-mgr 2024-04-26T04:46:24.742 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-904.gd8c6b0a3.el9.n 6/7 2024-04-26T04:46:24.782 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 7/7 2024-04-26T04:46:25.055 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:02 ago on Fri 26 Apr 2024 04:45:23 AM UTC. 2024-04-26T04:46:25.135 INFO:teuthology.orchestra.run.smithi012.stdout:Package ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:46:25.178 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-04-26T04:46:25.179 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2024-04-26T04:46:25.179 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-04-26T04:46:25.240 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-mgr-dashboard 2024-04-26T04:46:25.773 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:02 ago on Fri 26 Apr 2024 04:45:23 AM UTC. 2024-04-26T04:46:25.855 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 7/7 2024-04-26T04:46:25.855 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-904.gd8c6b0a3.el9.n 1/7 2024-04-26T04:46:25.855 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 2/7 2024-04-26T04:46:25.855 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-904.gd8c6b0a3.el9.no 3/7 2024-04-26T04:46:25.855 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 4/7 2024-04-26T04:46:25.855 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 5/7 2024-04-26T04:46:25.855 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 6/7 2024-04-26T04:46:25.899 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-04-26T04:46:25.900 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:46:25.900 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2024-04-26T04:46:25.900 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:46:25.900 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-04-26T04:46:25.900 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-dashboard noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 3.4 M 2024-04-26T04:46:25.900 INFO:teuthology.orchestra.run.smithi012.stdout:Installing dependencies: 2024-04-26T04:46:25.900 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-grafana-dashboards noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 24 k 2024-04-26T04:46:25.900 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-prometheus-alerts noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 15 k 2024-04-26T04:46:25.900 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-04-26T04:46:25.900 INFO:teuthology.orchestra.run.smithi012.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-04-26T04:46:25.901 INFO:teuthology.orchestra.run.smithi012.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-04-26T04:46:25.901 INFO:teuthology.orchestra.run.smithi012.stdout:Installing weak dependencies: 2024-04-26T04:46:25.901 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-04-26T04:46:25.901 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:25.901 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-04-26T04:46:25.901 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:46:25.901 INFO:teuthology.orchestra.run.smithi012.stdout:Install 7 Packages 2024-04-26T04:46:25.901 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:25.902 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 3.7 M 2024-04-26T04:46:25.902 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 80 M 2024-04-26T04:46:25.902 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-04-26T04:46:26.107 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 7/7 2024-04-26T04:46:26.108 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:26.108 INFO:teuthology.orchestra.run.smithi116.stdout:Installed: 2024-04-26T04:46:26.108 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-grafana-dashboards-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:46:26.108 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:46:26.108 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-prometheus-alerts-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:46:26.108 INFO:teuthology.orchestra.run.smithi116.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-04-26T04:46:26.108 INFO:teuthology.orchestra.run.smithi116.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-04-26T04:46:26.108 INFO:teuthology.orchestra.run.smithi116.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-04-26T04:46:26.108 INFO:teuthology.orchestra.run.smithi116.stdout: python3-routes-2.5.1-5.el9.noarch 2024-04-26T04:46:26.108 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:26.108 INFO:teuthology.orchestra.run.smithi116.stdout:Complete! 2024-04-26T04:46:26.253 INFO:teuthology.orchestra.run.smithi012.stdout:(1/7): ceph-grafana-dashboards-17.2.7-904.gd8c6 124 kB/s | 24 kB 00:00 2024-04-26T04:46:26.336 DEBUG:teuthology.orchestra.run.smithi116:> sudo yum -y install ceph-mgr-diskprediction-local 2024-04-26T04:46:26.345 INFO:teuthology.orchestra.run.smithi012.stdout:(2/7): python3-jwt+crypto-2.4.0-1.el9.noarch.rp 99 kB/s | 9.0 kB 00:00 2024-04-26T04:46:26.370 INFO:teuthology.orchestra.run.smithi012.stdout:(3/7): ceph-prometheus-alerts-17.2.7-904.gd8c6b 46 kB/s | 15 kB 00:00 2024-04-26T04:46:26.454 INFO:teuthology.orchestra.run.smithi012.stdout:(4/7): python3-jwt-2.4.0-1.el9.noarch.rpm 377 kB/s | 41 kB 00:00 2024-04-26T04:46:26.521 INFO:teuthology.orchestra.run.smithi012.stdout:(5/7): ceph-mgr-dashboard-17.2.7-904.gd8c6b0a3. 7.2 MB/s | 3.4 MB 00:00 2024-04-26T04:46:26.546 INFO:teuthology.orchestra.run.smithi012.stdout:(6/7): python3-repoze-lru-0.7-16.el9.noarch.rpm 175 kB/s | 31 kB 00:00 2024-04-26T04:46:26.613 INFO:teuthology.orchestra.run.smithi012.stdout:(7/7): python3-routes-2.5.1-5.el9.noarch.rpm 1.2 MB/s | 188 kB 00:00 2024-04-26T04:46:26.614 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:46:26.615 INFO:teuthology.orchestra.run.smithi012.stdout:Total 5.1 MB/s | 3.7 MB 00:00 2024-04-26T04:46:26.615 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-04-26T04:46:26.624 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-04-26T04:46:26.624 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-04-26T04:46:26.689 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-04-26T04:46:26.691 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-04-26T04:46:26.846 INFO:teuthology.orchestra.run.smithi116.stdout:Last metadata expiration check: 0:01:04 ago on Fri 26 Apr 2024 04:45:22 AM UTC. 2024-04-26T04:46:26.970 INFO:teuthology.orchestra.run.smithi116.stdout:Dependencies resolved. 2024-04-26T04:46:26.970 INFO:teuthology.orchestra.run.smithi116.stdout:====================================================================================== 2024-04-26T04:46:26.970 INFO:teuthology.orchestra.run.smithi116.stdout: Package Arch Version Repository Size 2024-04-26T04:46:26.970 INFO:teuthology.orchestra.run.smithi116.stdout:====================================================================================== 2024-04-26T04:46:26.970 INFO:teuthology.orchestra.run.smithi116.stdout:Installing: 2024-04-26T04:46:26.970 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 7.4 M 2024-04-26T04:46:26.971 INFO:teuthology.orchestra.run.smithi116.stdout:Installing dependencies: 2024-04-26T04:46:26.971 INFO:teuthology.orchestra.run.smithi116.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-04-26T04:46:26.971 INFO:teuthology.orchestra.run.smithi116.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-04-26T04:46:26.971 INFO:teuthology.orchestra.run.smithi116.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-04-26T04:46:26.971 INFO:teuthology.orchestra.run.smithi116.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-04-26T04:46:26.971 INFO:teuthology.orchestra.run.smithi116.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-04-26T04:46:26.971 INFO:teuthology.orchestra.run.smithi116.stdout: openblas x86_64 0.3.21-2.el9 appstream 35 k 2024-04-26T04:46:26.971 INFO:teuthology.orchestra.run.smithi116.stdout: openblas-openmp x86_64 0.3.21-2.el9 appstream 4.7 M 2024-04-26T04:46:26.971 INFO:teuthology.orchestra.run.smithi116.stdout: python3-devel x86_64 3.9.18-3.el9 appstream 244 k 2024-04-26T04:46:26.971 INFO:teuthology.orchestra.run.smithi116.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-04-26T04:46:26.971 INFO:teuthology.orchestra.run.smithi116.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-04-26T04:46:26.971 INFO:teuthology.orchestra.run.smithi116.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-04-26T04:46:26.971 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:26.971 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction Summary 2024-04-26T04:46:26.971 INFO:teuthology.orchestra.run.smithi116.stdout:====================================================================================== 2024-04-26T04:46:26.972 INFO:teuthology.orchestra.run.smithi116.stdout:Install 12 Packages 2024-04-26T04:46:26.972 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:26.972 INFO:teuthology.orchestra.run.smithi116.stdout:Total download size: 38 M 2024-04-26T04:46:26.972 INFO:teuthology.orchestra.run.smithi116.stdout:Installed size: 209 M 2024-04-26T04:46:26.972 INFO:teuthology.orchestra.run.smithi116.stdout:Downloading Packages: 2024-04-26T04:46:26.996 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-04-26T04:46:27.056 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 1/7 2024-04-26T04:46:27.090 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 2/7 2024-04-26T04:46:27.137 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/7 2024-04-26T04:46:27.178 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/7 2024-04-26T04:46:27.206 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-904.gd8c6b0a3.el9.no 5/7 2024-04-26T04:46:27.593 INFO:teuthology.orchestra.run.smithi116.stdout:(1/12): libgfortran-11.4.1-3.el9.x86_64.rpm 4.1 MB/s | 803 kB 00:00 2024-04-26T04:46:27.635 INFO:teuthology.orchestra.run.smithi116.stdout:(2/12): flexiblas-3.0.4-8.el9.x86_64.rpm 785 kB/s | 33 kB 00:00 2024-04-26T04:46:27.760 INFO:teuthology.orchestra.run.smithi116.stdout:(3/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 24 MB/s | 3.0 MB 00:00 2024-04-26T04:46:27.803 INFO:teuthology.orchestra.run.smithi116.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 436 kB/s | 18 kB 00:00 2024-04-26T04:46:27.836 INFO:teuthology.orchestra.run.smithi116.stdout:(5/12): openblas-0.3.21-2.el9.x86_64.rpm 1.0 MB/s | 35 kB 00:00 2024-04-26T04:46:28.011 INFO:teuthology.orchestra.run.smithi116.stdout:(6/12): openblas-openmp-0.3.21-2.el9.x86_64.rpm 27 MB/s | 4.7 MB 00:00 2024-04-26T04:46:28.103 INFO:teuthology.orchestra.run.smithi116.stdout:(7/12): ceph-mgr-diskprediction-local-17.2.7-90 11 MB/s | 7.4 MB 00:00 2024-04-26T04:46:28.128 INFO:teuthology.orchestra.run.smithi116.stdout:(8/12): python3-devel-3.9.18-3.el9.x86_64.rpm 2.0 MB/s | 244 kB 00:00 2024-04-26T04:46:28.196 INFO:teuthology.orchestra.run.smithi116.stdout:(9/12): python3-numpy-f2py-1.20.1-5.el9.x86_64. 3.7 MB/s | 250 kB 00:00 2024-04-26T04:46:28.255 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-904.gd8c6b0a3.el9.n 6/7 2024-04-26T04:46:28.292 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 7/7 2024-04-26T04:46:28.371 INFO:teuthology.orchestra.run.smithi116.stdout:(10/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 19 MB/s | 5.1 MB 00:00 2024-04-26T04:46:28.430 INFO:teuthology.orchestra.run.smithi116.stdout:(11/12): libquadmath-11.4.1-3.el9.x86_64.rpm 190 kB/s | 195 kB 00:01 2024-04-26T04:46:28.905 INFO:teuthology.orchestra.run.smithi116.stdout:(12/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 23 MB/s | 16 MB 00:00 2024-04-26T04:46:28.908 INFO:teuthology.orchestra.run.smithi116.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:46:28.908 INFO:teuthology.orchestra.run.smithi116.stdout:Total 20 MB/s | 38 MB 00:01 2024-04-26T04:46:29.256 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction check 2024-04-26T04:46:29.284 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction check succeeded. 2024-04-26T04:46:29.284 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction test 2024-04-26T04:46:29.381 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 7/7 2024-04-26T04:46:29.381 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-904.gd8c6b0a3.el9.n 1/7 2024-04-26T04:46:29.381 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 2/7 2024-04-26T04:46:29.381 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-904.gd8c6b0a3.el9.no 3/7 2024-04-26T04:46:29.381 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 4/7 2024-04-26T04:46:29.382 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 5/7 2024-04-26T04:46:29.382 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 6/7 2024-04-26T04:46:29.644 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 7/7 2024-04-26T04:46:29.644 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:29.645 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-04-26T04:46:29.645 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-grafana-dashboards-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:46:29.645 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:46:29.645 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-prometheus-alerts-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:46:29.645 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-04-26T04:46:29.645 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-04-26T04:46:29.645 INFO:teuthology.orchestra.run.smithi012.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-04-26T04:46:29.645 INFO:teuthology.orchestra.run.smithi012.stdout: python3-routes-2.5.1-5.el9.noarch 2024-04-26T04:46:29.645 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:29.645 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-04-26T04:46:29.688 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction test succeeded. 2024-04-26T04:46:29.688 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction 2024-04-26T04:46:29.818 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-mgr-diskprediction-local 2024-04-26T04:46:30.354 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:07 ago on Fri 26 Apr 2024 04:45:23 AM UTC. 2024-04-26T04:46:30.435 INFO:teuthology.orchestra.run.smithi116.stdout: Preparing : 1/1 2024-04-26T04:46:30.481 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-04-26T04:46:30.482 INFO:teuthology.orchestra.run.smithi012.stdout:====================================================================================== 2024-04-26T04:46:30.482 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2024-04-26T04:46:30.482 INFO:teuthology.orchestra.run.smithi012.stdout:====================================================================================== 2024-04-26T04:46:30.482 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-04-26T04:46:30.482 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 7.4 M 2024-04-26T04:46:30.482 INFO:teuthology.orchestra.run.smithi012.stdout:Installing dependencies: 2024-04-26T04:46:30.482 INFO:teuthology.orchestra.run.smithi012.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-04-26T04:46:30.482 INFO:teuthology.orchestra.run.smithi012.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-04-26T04:46:30.482 INFO:teuthology.orchestra.run.smithi012.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-04-26T04:46:30.483 INFO:teuthology.orchestra.run.smithi012.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-04-26T04:46:30.483 INFO:teuthology.orchestra.run.smithi012.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-04-26T04:46:30.483 INFO:teuthology.orchestra.run.smithi012.stdout: openblas x86_64 0.3.21-2.el9 appstream 35 k 2024-04-26T04:46:30.483 INFO:teuthology.orchestra.run.smithi012.stdout: openblas-openmp x86_64 0.3.21-2.el9 appstream 4.7 M 2024-04-26T04:46:30.483 INFO:teuthology.orchestra.run.smithi012.stdout: python3-devel x86_64 3.9.18-3.el9 appstream 244 k 2024-04-26T04:46:30.483 INFO:teuthology.orchestra.run.smithi012.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-04-26T04:46:30.483 INFO:teuthology.orchestra.run.smithi012.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-04-26T04:46:30.483 INFO:teuthology.orchestra.run.smithi012.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-04-26T04:46:30.483 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:30.483 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-04-26T04:46:30.484 INFO:teuthology.orchestra.run.smithi012.stdout:====================================================================================== 2024-04-26T04:46:30.484 INFO:teuthology.orchestra.run.smithi012.stdout:Install 12 Packages 2024-04-26T04:46:30.484 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:30.484 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 38 M 2024-04-26T04:46:30.484 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 209 M 2024-04-26T04:46:30.484 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-04-26T04:46:30.518 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-04-26T04:46:30.570 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 2/12 2024-04-26T04:46:30.678 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 3/12 2024-04-26T04:46:30.722 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-devel-3.9.18-3.el9.x86_64 4/12 2024-04-26T04:46:30.895 INFO:teuthology.orchestra.run.smithi012.stdout:(1/12): libquadmath-11.4.1-3.el9.x86_64.rpm 966 kB/s | 195 kB 00:00 2024-04-26T04:46:30.898 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : openblas-0.3.21-2.el9.x86_64 5/12 2024-04-26T04:46:30.936 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-04-26T04:46:30.937 INFO:teuthology.orchestra.run.smithi012.stdout:(2/12): libgfortran-11.4.1-3.el9.x86_64.rpm 3.2 MB/s | 803 kB 00:00 2024-04-26T04:46:30.962 INFO:teuthology.orchestra.run.smithi012.stdout:(3/12): flexiblas-3.0.4-8.el9.x86_64.rpm 492 kB/s | 33 kB 00:00 2024-04-26T04:46:31.013 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-04-26T04:46:31.113 INFO:teuthology.orchestra.run.smithi012.stdout:(4/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 17 MB/s | 3.0 MB 00:00 2024-04-26T04:46:31.130 INFO:teuthology.orchestra.run.smithi012.stdout:(5/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 109 kB/s | 18 kB 00:00 2024-04-26T04:46:31.147 INFO:teuthology.orchestra.run.smithi012.stdout:(6/12): openblas-0.3.21-2.el9.x86_64.rpm 1.0 MB/s | 35 kB 00:00 2024-04-26T04:46:31.305 INFO:teuthology.orchestra.run.smithi012.stdout:(7/12): ceph-mgr-diskprediction-local-17.2.7-90 12 MB/s | 7.4 MB 00:00 2024-04-26T04:46:31.372 INFO:teuthology.orchestra.run.smithi012.stdout:(8/12): openblas-openmp-0.3.21-2.el9.x86_64.rpm 20 MB/s | 4.7 MB 00:00 2024-04-26T04:46:31.397 INFO:teuthology.orchestra.run.smithi012.stdout:(9/12): python3-devel-3.9.18-3.el9.x86_64.rpm 975 kB/s | 244 kB 00:00 2024-04-26T04:46:31.481 INFO:teuthology.orchestra.run.smithi012.stdout:(10/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 29 MB/s | 5.1 MB 00:00 2024-04-26T04:46:31.506 INFO:teuthology.orchestra.run.smithi012.stdout:(11/12): python3-numpy-f2py-1.20.1-5.el9.x86_64 2.3 MB/s | 250 kB 00:00 2024-04-26T04:46:31.652 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-04-26T04:46:31.774 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-04-26T04:46:31.848 INFO:teuthology.orchestra.run.smithi012.stdout:(12/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 36 MB/s | 16 MB 00:00 2024-04-26T04:46:31.851 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:46:31.851 INFO:teuthology.orchestra.run.smithi012.stdout:Total 28 MB/s | 38 MB 00:01 2024-04-26T04:46:32.213 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-04-26T04:46:32.242 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-04-26T04:46:32.242 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-04-26T04:46:32.676 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-04-26T04:46:32.676 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-04-26T04:46:33.001 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-04-26T04:46:33.457 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-04-26T04:46:33.520 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-04-26T04:46:33.563 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 2/12 2024-04-26T04:46:33.668 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 3/12 2024-04-26T04:46:33.706 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-devel-3.9.18-3.el9.x86_64 4/12 2024-04-26T04:46:33.891 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : openblas-0.3.21-2.el9.x86_64 5/12 2024-04-26T04:46:33.921 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-04-26T04:46:34.001 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-04-26T04:46:34.038 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-04-26T04:46:34.060 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0 12/12 2024-04-26T04:46:34.686 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-04-26T04:46:34.805 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-04-26T04:46:36.136 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-04-26T04:46:36.533 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0 12/12 2024-04-26T04:46:36.534 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0 1/12 2024-04-26T04:46:36.534 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 2/12 2024-04-26T04:46:36.534 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 3/12 2024-04-26T04:46:36.534 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-04-26T04:46:36.534 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-04-26T04:46:36.534 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-04-26T04:46:36.534 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 7/12 2024-04-26T04:46:36.534 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 8/12 2024-04-26T04:46:36.534 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-devel-3.9.18-3.el9.x86_64 9/12 2024-04-26T04:46:36.534 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-04-26T04:46:36.534 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-04-26T04:46:36.901 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-04-26T04:46:36.901 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:36.901 INFO:teuthology.orchestra.run.smithi116.stdout:Installed: 2024-04-26T04:46:36.901 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:46:36.901 INFO:teuthology.orchestra.run.smithi116.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-04-26T04:46:36.902 INFO:teuthology.orchestra.run.smithi116.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-04-26T04:46:36.902 INFO:teuthology.orchestra.run.smithi116.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-04-26T04:46:36.902 INFO:teuthology.orchestra.run.smithi116.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-04-26T04:46:36.902 INFO:teuthology.orchestra.run.smithi116.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-04-26T04:46:36.902 INFO:teuthology.orchestra.run.smithi116.stdout: openblas-0.3.21-2.el9.x86_64 2024-04-26T04:46:36.902 INFO:teuthology.orchestra.run.smithi116.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-04-26T04:46:36.902 INFO:teuthology.orchestra.run.smithi116.stdout: python3-devel-3.9.18-3.el9.x86_64 2024-04-26T04:46:36.902 INFO:teuthology.orchestra.run.smithi116.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-04-26T04:46:36.902 INFO:teuthology.orchestra.run.smithi116.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-04-26T04:46:36.903 INFO:teuthology.orchestra.run.smithi116.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-04-26T04:46:36.903 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:36.903 INFO:teuthology.orchestra.run.smithi116.stdout:Complete! 2024-04-26T04:46:37.217 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-04-26T04:46:37.246 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0 12/12 2024-04-26T04:46:37.313 DEBUG:teuthology.orchestra.run.smithi116:> sudo yum -y install ceph-mgr-rook 2024-04-26T04:46:37.832 INFO:teuthology.orchestra.run.smithi116.stdout:Last metadata expiration check: 0:01:15 ago on Fri 26 Apr 2024 04:45:22 AM UTC. 2024-04-26T04:46:37.953 INFO:teuthology.orchestra.run.smithi116.stdout:Dependencies resolved. 2024-04-26T04:46:37.954 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:46:37.954 INFO:teuthology.orchestra.run.smithi116.stdout: Package Arch Version Repository Size 2024-04-26T04:46:37.954 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:46:37.954 INFO:teuthology.orchestra.run.smithi116.stdout:Installing: 2024-04-26T04:46:37.954 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-mgr-rook noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 47 k 2024-04-26T04:46:37.954 INFO:teuthology.orchestra.run.smithi116.stdout:Installing dependencies: 2024-04-26T04:46:37.954 INFO:teuthology.orchestra.run.smithi116.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-04-26T04:46:37.955 INFO:teuthology.orchestra.run.smithi116.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-04-26T04:46:37.955 INFO:teuthology.orchestra.run.smithi116.stdout: python3-google-auth noarch 1:2.29.0-1.el9 epel 206 k 2024-04-26T04:46:37.955 INFO:teuthology.orchestra.run.smithi116.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-04-26T04:46:37.955 INFO:teuthology.orchestra.run.smithi116.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-04-26T04:46:37.955 INFO:teuthology.orchestra.run.smithi116.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-04-26T04:46:37.955 INFO:teuthology.orchestra.run.smithi116.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-04-26T04:46:37.955 INFO:teuthology.orchestra.run.smithi116.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-04-26T04:46:37.955 INFO:teuthology.orchestra.run.smithi116.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-04-26T04:46:37.955 INFO:teuthology.orchestra.run.smithi116.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-04-26T04:46:37.955 INFO:teuthology.orchestra.run.smithi116.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-04-26T04:46:37.955 INFO:teuthology.orchestra.run.smithi116.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-04-26T04:46:37.955 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:37.955 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction Summary 2024-04-26T04:46:37.955 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:46:37.956 INFO:teuthology.orchestra.run.smithi116.stdout:Install 13 Packages 2024-04-26T04:46:37.956 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:37.956 INFO:teuthology.orchestra.run.smithi116.stdout:Total download size: 2.2 M 2024-04-26T04:46:37.956 INFO:teuthology.orchestra.run.smithi116.stdout:Installed size: 26 M 2024-04-26T04:46:37.956 INFO:teuthology.orchestra.run.smithi116.stdout:Downloading Packages: 2024-04-26T04:46:38.236 INFO:teuthology.orchestra.run.smithi116.stdout:(1/13): python3-jsonpointer-2.0-4.el9.noarch.rp 184 kB/s | 19 kB 00:00 2024-04-26T04:46:38.336 INFO:teuthology.orchestra.run.smithi116.stdout:(2/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 2.2 MB/s | 222 kB 00:00 2024-04-26T04:46:38.361 INFO:teuthology.orchestra.run.smithi116.stdout:(3/13): ceph-mgr-rook-17.2.7-904.gd8c6b0a3.el9. 207 kB/s | 47 kB 00:00 2024-04-26T04:46:38.387 INFO:teuthology.orchestra.run.smithi116.stdout:(4/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 3.1 MB/s | 159 kB 00:00 2024-04-26T04:46:38.420 INFO:teuthology.orchestra.run.smithi116.stdout:(5/13): python3-pyasn1-modules-0.4.8-6.el9.noar 4.7 MB/s | 279 kB 00:00 2024-04-26T04:46:38.445 INFO:teuthology.orchestra.run.smithi116.stdout:(6/13): python3-requests-oauthlib-1.3.0-12.el9. 917 kB/s | 54 kB 00:00 2024-04-26T04:46:38.504 INFO:teuthology.orchestra.run.smithi116.stdout:(7/13): python3-certifi-2021.10.8-2.el9.noarch. 255 kB/s | 15 kB 00:00 2024-04-26T04:46:38.530 INFO:teuthology.orchestra.run.smithi116.stdout:(8/13): python3-cachetools-4.2.4-1.el9.noarch.r 295 kB/s | 32 kB 00:00 2024-04-26T04:46:38.597 INFO:teuthology.orchestra.run.smithi116.stdout:(9/13): python3-google-auth-2.29.0-1.el9.noarch 2.2 MB/s | 206 kB 00:00 2024-04-26T04:46:38.639 INFO:teuthology.orchestra.run.smithi116.stdout:(10/13): python3-rsa-4.9-2.el9.noarch.rpm 1.4 MB/s | 59 kB 00:00 2024-04-26T04:46:38.672 INFO:teuthology.orchestra.run.smithi116.stdout:(11/13): python3-kubernetes-26.1.0-2.el9.noarch 7.2 MB/s | 1.0 MB 00:00 2024-04-26T04:46:38.697 INFO:teuthology.orchestra.run.smithi116.stdout:(12/13): python3-jsonpatch-1.21-16.el9.noarch.r 45 kB/s | 26 kB 00:00 2024-04-26T04:46:38.723 INFO:teuthology.orchestra.run.smithi116.stdout:(13/13): python3-websocket-client-1.2.3-2.el9.n 1.0 MB/s | 90 kB 00:00 2024-04-26T04:46:38.724 INFO:teuthology.orchestra.run.smithi116.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:46:38.725 INFO:teuthology.orchestra.run.smithi116.stdout:Total 2.9 MB/s | 2.2 MB 00:00 2024-04-26T04:46:38.786 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction check 2024-04-26T04:46:38.801 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction check succeeded. 2024-04-26T04:46:38.801 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction test 2024-04-26T04:46:38.885 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction test succeeded. 2024-04-26T04:46:38.885 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction 2024-04-26T04:46:39.462 INFO:teuthology.orchestra.run.smithi116.stdout: Preparing : 1/1 2024-04-26T04:46:39.631 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-04-26T04:46:39.691 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-04-26T04:46:39.701 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0 12/12 2024-04-26T04:46:39.701 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0 1/12 2024-04-26T04:46:39.701 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 2/12 2024-04-26T04:46:39.701 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 3/12 2024-04-26T04:46:39.701 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-04-26T04:46:39.701 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-04-26T04:46:39.701 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-04-26T04:46:39.701 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 7/12 2024-04-26T04:46:39.701 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 8/12 2024-04-26T04:46:39.701 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-devel-3.9.18-3.el9.x86_64 9/12 2024-04-26T04:46:39.702 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-04-26T04:46:39.702 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-04-26T04:46:39.755 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-04-26T04:46:39.792 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-04-26T04:46:39.833 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 5/13 2024-04-26T04:46:39.927 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-04-26T04:46:40.029 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-google-auth-1:2.29.0-1.el9.noarch 7/13 2024-04-26T04:46:40.052 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-04-26T04:46:40.052 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:40.052 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-04-26T04:46:40.052 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:46:40.052 INFO:teuthology.orchestra.run.smithi012.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-04-26T04:46:40.052 INFO:teuthology.orchestra.run.smithi012.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-04-26T04:46:40.052 INFO:teuthology.orchestra.run.smithi012.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-04-26T04:46:40.052 INFO:teuthology.orchestra.run.smithi012.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-04-26T04:46:40.052 INFO:teuthology.orchestra.run.smithi012.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-04-26T04:46:40.052 INFO:teuthology.orchestra.run.smithi012.stdout: openblas-0.3.21-2.el9.x86_64 2024-04-26T04:46:40.052 INFO:teuthology.orchestra.run.smithi012.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-04-26T04:46:40.052 INFO:teuthology.orchestra.run.smithi012.stdout: python3-devel-3.9.18-3.el9.x86_64 2024-04-26T04:46:40.052 INFO:teuthology.orchestra.run.smithi012.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-04-26T04:46:40.053 INFO:teuthology.orchestra.run.smithi012.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-04-26T04:46:40.053 INFO:teuthology.orchestra.run.smithi012.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-04-26T04:46:40.053 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:40.053 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-04-26T04:46:40.077 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-04-26T04:46:40.417 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-mgr-rook 2024-04-26T04:46:40.787 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-04-26T04:46:40.832 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-04-26T04:46:40.867 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-04-26T04:46:40.911 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-04-26T04:46:40.936 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 13/13 2024-04-26T04:46:40.952 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:17 ago on Fri 26 Apr 2024 04:45:23 AM UTC. 2024-04-26T04:46:41.078 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-04-26T04:46:41.079 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:46:41.080 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2024-04-26T04:46:41.080 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:46:41.080 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-04-26T04:46:41.080 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-rook noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 47 k 2024-04-26T04:46:41.080 INFO:teuthology.orchestra.run.smithi012.stdout:Installing dependencies: 2024-04-26T04:46:41.080 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-04-26T04:46:41.080 INFO:teuthology.orchestra.run.smithi012.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-04-26T04:46:41.080 INFO:teuthology.orchestra.run.smithi012.stdout: python3-google-auth noarch 1:2.29.0-1.el9 epel 206 k 2024-04-26T04:46:41.080 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-04-26T04:46:41.080 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-04-26T04:46:41.080 INFO:teuthology.orchestra.run.smithi012.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-04-26T04:46:41.080 INFO:teuthology.orchestra.run.smithi012.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-04-26T04:46:41.081 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-04-26T04:46:41.081 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-04-26T04:46:41.081 INFO:teuthology.orchestra.run.smithi012.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-04-26T04:46:41.081 INFO:teuthology.orchestra.run.smithi012.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-04-26T04:46:41.081 INFO:teuthology.orchestra.run.smithi012.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-04-26T04:46:41.081 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:41.081 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-04-26T04:46:41.081 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:46:41.081 INFO:teuthology.orchestra.run.smithi012.stdout:Install 13 Packages 2024-04-26T04:46:41.081 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:41.082 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 2.2 M 2024-04-26T04:46:41.082 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 26 M 2024-04-26T04:46:41.082 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-04-26T04:46:41.662 INFO:teuthology.orchestra.run.smithi012.stdout:(1/13): python3-jsonpatch-1.21-16.el9.noarch.rp 168 kB/s | 26 kB 00:00 2024-04-26T04:46:41.687 INFO:teuthology.orchestra.run.smithi012.stdout:(2/13): python3-jsonpointer-2.0-4.el9.noarch.rp 108 kB/s | 19 kB 00:00 2024-04-26T04:46:41.702 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 13/13 2024-04-26T04:46:41.702 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/13 2024-04-26T04:46:41.702 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-04-26T04:46:41.703 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-04-26T04:46:41.703 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-04-26T04:46:41.703 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-04-26T04:46:41.703 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-04-26T04:46:41.703 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-04-26T04:46:41.703 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-04-26T04:46:41.703 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 9/13 2024-04-26T04:46:41.703 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-google-auth-1:2.29.0-1.el9.noarch 10/13 2024-04-26T04:46:41.703 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-04-26T04:46:41.703 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-04-26T04:46:41.771 INFO:teuthology.orchestra.run.smithi012.stdout:(3/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 2.6 MB/s | 222 kB 00:00 2024-04-26T04:46:41.796 INFO:teuthology.orchestra.run.smithi012.stdout:(4/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.4 MB/s | 159 kB 00:00 2024-04-26T04:46:41.829 INFO:teuthology.orchestra.run.smithi012.stdout:(5/13): python3-pyasn1-modules-0.4.8-6.el9.noar 4.7 MB/s | 279 kB 00:00 2024-04-26T04:46:41.855 INFO:teuthology.orchestra.run.smithi012.stdout:(6/13): python3-requests-oauthlib-1.3.0-12.el9. 921 kB/s | 54 kB 00:00 2024-04-26T04:46:41.880 INFO:teuthology.orchestra.run.smithi012.stdout:(7/13): ceph-mgr-rook-17.2.7-904.gd8c6b0a3.el9. 128 kB/s | 47 kB 00:00 2024-04-26T04:46:41.972 INFO:teuthology.orchestra.run.smithi012.stdout:(8/13): python3-certifi-2021.10.8-2.el9.noarch. 128 kB/s | 15 kB 00:00 2024-04-26T04:46:41.998 INFO:teuthology.orchestra.run.smithi012.stdout:(9/13): python3-cachetools-4.2.4-1.el9.noarch.r 192 kB/s | 32 kB 00:00 2024-04-26T04:46:42.017 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-04-26T04:46:42.017 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:42.017 INFO:teuthology.orchestra.run.smithi116.stdout:Installed: 2024-04-26T04:46:42.017 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:46:42.018 INFO:teuthology.orchestra.run.smithi116.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-04-26T04:46:42.018 INFO:teuthology.orchestra.run.smithi116.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-04-26T04:46:42.018 INFO:teuthology.orchestra.run.smithi116.stdout: python3-google-auth-1:2.29.0-1.el9.noarch 2024-04-26T04:46:42.018 INFO:teuthology.orchestra.run.smithi116.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-04-26T04:46:42.018 INFO:teuthology.orchestra.run.smithi116.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-04-26T04:46:42.018 INFO:teuthology.orchestra.run.smithi116.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-04-26T04:46:42.018 INFO:teuthology.orchestra.run.smithi116.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-04-26T04:46:42.018 INFO:teuthology.orchestra.run.smithi116.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-04-26T04:46:42.018 INFO:teuthology.orchestra.run.smithi116.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-04-26T04:46:42.018 INFO:teuthology.orchestra.run.smithi116.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-04-26T04:46:42.018 INFO:teuthology.orchestra.run.smithi116.stdout: python3-rsa-4.9-2.el9.noarch 2024-04-26T04:46:42.018 INFO:teuthology.orchestra.run.smithi116.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-04-26T04:46:42.019 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:42.019 INFO:teuthology.orchestra.run.smithi116.stdout:Complete! 2024-04-26T04:46:42.090 INFO:teuthology.orchestra.run.smithi012.stdout:(10/13): python3-google-auth-2.29.0-1.el9.noarc 985 kB/s | 206 kB 00:00 2024-04-26T04:46:42.115 INFO:teuthology.orchestra.run.smithi012.stdout:(11/13): python3-rsa-4.9-2.el9.noarch.rpm 505 kB/s | 59 kB 00:00 2024-04-26T04:46:42.173 INFO:teuthology.orchestra.run.smithi012.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 1.0 MB/s | 90 kB 00:00 2024-04-26T04:46:42.248 INFO:teuthology.orchestra.run.smithi012.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 3.7 MB/s | 1.0 MB 00:00 2024-04-26T04:46:42.250 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:46:42.251 INFO:teuthology.orchestra.run.smithi012.stdout:Total 1.9 MB/s | 2.2 MB 00:01 2024-04-26T04:46:42.262 DEBUG:teuthology.orchestra.run.smithi116:> sudo yum -y install ceph-mgr-cephadm 2024-04-26T04:46:42.316 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-04-26T04:46:42.332 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-04-26T04:46:42.333 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-04-26T04:46:42.424 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-04-26T04:46:42.425 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-04-26T04:46:42.766 INFO:teuthology.orchestra.run.smithi116.stdout:Last metadata expiration check: 0:01:20 ago on Fri 26 Apr 2024 04:45:22 AM UTC. 2024-04-26T04:46:42.885 INFO:teuthology.orchestra.run.smithi116.stdout:Dependencies resolved. 2024-04-26T04:46:42.886 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:46:42.886 INFO:teuthology.orchestra.run.smithi116.stdout: Package Arch Version Repository Size 2024-04-26T04:46:42.886 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:46:42.887 INFO:teuthology.orchestra.run.smithi116.stdout:Installing: 2024-04-26T04:46:42.888 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-mgr-cephadm noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 123 k 2024-04-26T04:46:42.888 INFO:teuthology.orchestra.run.smithi116.stdout:Installing dependencies: 2024-04-26T04:46:42.888 INFO:teuthology.orchestra.run.smithi116.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-04-26T04:46:42.888 INFO:teuthology.orchestra.run.smithi116.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-04-26T04:46:42.888 INFO:teuthology.orchestra.run.smithi116.stdout: python3-jinja2 noarch 2.11.3-5.el9 appstream 248 k 2024-04-26T04:46:42.888 INFO:teuthology.orchestra.run.smithi116.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-04-26T04:46:42.889 INFO:teuthology.orchestra.run.smithi116.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-04-26T04:46:42.889 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:42.889 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction Summary 2024-04-26T04:46:42.889 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:46:42.889 INFO:teuthology.orchestra.run.smithi116.stdout:Install 6 Packages 2024-04-26T04:46:42.889 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:42.890 INFO:teuthology.orchestra.run.smithi116.stdout:Total download size: 7.0 M 2024-04-26T04:46:42.890 INFO:teuthology.orchestra.run.smithi116.stdout:Installed size: 33 M 2024-04-26T04:46:42.890 INFO:teuthology.orchestra.run.smithi116.stdout:Downloading Packages: 2024-04-26T04:46:43.027 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-04-26T04:46:43.208 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-04-26T04:46:43.268 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-04-26T04:46:43.325 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-04-26T04:46:43.368 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-04-26T04:46:43.410 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 5/13 2024-04-26T04:46:43.516 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-04-26T04:46:43.626 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-google-auth-1:2.29.0-1.el9.noarch 7/13 2024-04-26T04:46:43.633 INFO:teuthology.orchestra.run.smithi116.stdout:(1/6): python3-jinja2-2.11.3-5.el9.noarch.rpm 997 kB/s | 248 kB 00:00 2024-04-26T04:46:43.688 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-04-26T04:46:43.758 INFO:teuthology.orchestra.run.smithi116.stdout:(2/6): ceph-mgr-cephadm-17.2.7-904.gd8c6b0a3.el 330 kB/s | 123 kB 00:00 2024-04-26T04:46:44.075 INFO:teuthology.orchestra.run.smithi116.stdout:(3/6): python3-natsort-7.1.1-5.el9.noarch.rpm 183 kB/s | 58 kB 00:00 2024-04-26T04:46:44.176 INFO:teuthology.orchestra.run.smithi116.stdout:(4/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 1.0 MB/s | 548 kB 00:00 2024-04-26T04:46:44.201 INFO:teuthology.orchestra.run.smithi116.stdout:(5/6): python3-typing-extensions-4.4.0-2.el9.no 407 kB/s | 51 kB 00:00 2024-04-26T04:46:44.445 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-04-26T04:46:44.492 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-04-26T04:46:44.526 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-04-26T04:46:44.571 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-04-26T04:46:44.594 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 13/13 2024-04-26T04:46:45.296 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 13/13 2024-04-26T04:46:45.296 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/13 2024-04-26T04:46:45.296 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-04-26T04:46:45.296 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-04-26T04:46:45.296 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-04-26T04:46:45.296 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-04-26T04:46:45.296 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-04-26T04:46:45.296 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-04-26T04:46:45.296 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-04-26T04:46:45.297 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 9/13 2024-04-26T04:46:45.297 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-google-auth-1:2.29.0-1.el9.noarch 10/13 2024-04-26T04:46:45.297 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-04-26T04:46:45.297 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-04-26T04:46:45.418 INFO:teuthology.orchestra.run.smithi116.stdout:(6/6): python3-babel-2.9.1-2.el9.noarch.rpm 2.9 MB/s | 6.0 MB 00:02 2024-04-26T04:46:45.420 INFO:teuthology.orchestra.run.smithi116.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:46:45.420 INFO:teuthology.orchestra.run.smithi116.stdout:Total 2.8 MB/s | 7.0 MB 00:02 2024-04-26T04:46:45.491 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction check 2024-04-26T04:46:45.502 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction check succeeded. 2024-04-26T04:46:45.502 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction test 2024-04-26T04:46:45.598 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction test succeeded. 2024-04-26T04:46:45.602 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction 2024-04-26T04:46:45.609 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-04-26T04:46:45.609 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:45.609 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-04-26T04:46:45.609 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:46:45.609 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-04-26T04:46:45.609 INFO:teuthology.orchestra.run.smithi012.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-04-26T04:46:45.609 INFO:teuthology.orchestra.run.smithi012.stdout: python3-google-auth-1:2.29.0-1.el9.noarch 2024-04-26T04:46:45.609 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-04-26T04:46:45.609 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-04-26T04:46:45.610 INFO:teuthology.orchestra.run.smithi012.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-04-26T04:46:45.610 INFO:teuthology.orchestra.run.smithi012.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-04-26T04:46:45.610 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-04-26T04:46:45.610 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-04-26T04:46:45.610 INFO:teuthology.orchestra.run.smithi012.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-04-26T04:46:45.610 INFO:teuthology.orchestra.run.smithi012.stdout: python3-rsa-4.9-2.el9.noarch 2024-04-26T04:46:45.610 INFO:teuthology.orchestra.run.smithi012.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-04-26T04:46:45.610 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:45.610 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-04-26T04:46:45.827 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-mgr-cephadm 2024-04-26T04:46:45.908 INFO:teuthology.orchestra.run.smithi116.stdout: Preparing : 1/1 2024-04-26T04:46:46.078 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 1/6 2024-04-26T04:46:46.137 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-04-26T04:46:46.364 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:23 ago on Fri 26 Apr 2024 04:45:23 AM UTC. 2024-04-26T04:46:46.488 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-04-26T04:46:46.489 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:46:46.489 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2024-04-26T04:46:46.489 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:46:46.489 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-04-26T04:46:46.490 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-cephadm noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 123 k 2024-04-26T04:46:46.490 INFO:teuthology.orchestra.run.smithi012.stdout:Installing dependencies: 2024-04-26T04:46:46.490 INFO:teuthology.orchestra.run.smithi012.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-04-26T04:46:46.490 INFO:teuthology.orchestra.run.smithi012.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-04-26T04:46:46.490 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jinja2 noarch 2.11.3-5.el9 appstream 248 k 2024-04-26T04:46:46.490 INFO:teuthology.orchestra.run.smithi012.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-04-26T04:46:46.490 INFO:teuthology.orchestra.run.smithi012.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-04-26T04:46:46.490 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:46.490 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-04-26T04:46:46.491 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:46:46.491 INFO:teuthology.orchestra.run.smithi012.stdout:Install 6 Packages 2024-04-26T04:46:46.491 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:46.491 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 7.0 M 2024-04-26T04:46:46.491 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 33 M 2024-04-26T04:46:46.491 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-04-26T04:46:46.677 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-04-26T04:46:46.758 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-04-26T04:46:46.814 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 5/6 2024-04-26T04:46:46.842 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 6/6 2024-04-26T04:46:46.992 INFO:teuthology.orchestra.run.smithi012.stdout:(1/6): python3-babel-2.9.1-2.el9.noarch.rpm 21 MB/s | 6.0 MB 00:00 2024-04-26T04:46:47.267 INFO:teuthology.orchestra.run.smithi012.stdout:(2/6): ceph-mgr-cephadm-17.2.7-904.gd8c6b0a3.el 222 kB/s | 123 kB 00:00 2024-04-26T04:46:47.293 INFO:teuthology.orchestra.run.smithi012.stdout:(3/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 1.8 MB/s | 548 kB 00:00 2024-04-26T04:46:47.352 INFO:teuthology.orchestra.run.smithi012.stdout:(4/6): python3-typing-extensions-4.4.0-2.el9.no 875 kB/s | 51 kB 00:00 2024-04-26T04:46:47.418 INFO:teuthology.orchestra.run.smithi012.stdout:(5/6): python3-natsort-7.1.1-5.el9.noarch.rpm 383 kB/s | 58 kB 00:00 2024-04-26T04:46:47.542 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 6/6 2024-04-26T04:46:47.542 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/6 2024-04-26T04:46:47.543 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-04-26T04:46:47.543 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 3/6 2024-04-26T04:46:47.543 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-04-26T04:46:47.543 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-04-26T04:46:47.660 INFO:teuthology.orchestra.run.smithi012.stdout:(6/6): python3-jinja2-2.11.3-5.el9.noarch.rpm 261 kB/s | 248 kB 00:00 2024-04-26T04:46:47.663 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:46:47.663 INFO:teuthology.orchestra.run.smithi012.stdout:Total 5.9 MB/s | 7.0 MB 00:01 2024-04-26T04:46:47.738 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-04-26T04:46:47.751 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-04-26T04:46:47.752 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-04-26T04:46:47.799 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 6/6 2024-04-26T04:46:47.799 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:47.799 INFO:teuthology.orchestra.run.smithi116.stdout:Installed: 2024-04-26T04:46:47.799 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:46:47.799 INFO:teuthology.orchestra.run.smithi116.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-04-26T04:46:47.799 INFO:teuthology.orchestra.run.smithi116.stdout: python3-babel-2.9.1-2.el9.noarch 2024-04-26T04:46:47.799 INFO:teuthology.orchestra.run.smithi116.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-04-26T04:46:47.800 INFO:teuthology.orchestra.run.smithi116.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-04-26T04:46:47.800 INFO:teuthology.orchestra.run.smithi116.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-04-26T04:46:47.800 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:47.800 INFO:teuthology.orchestra.run.smithi116.stdout:Complete! 2024-04-26T04:46:47.853 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-04-26T04:46:47.853 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-04-26T04:46:48.042 DEBUG:teuthology.orchestra.run.smithi116:> sudo yum -y install ceph-fuse 2024-04-26T04:46:48.167 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-04-26T04:46:48.356 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 1/6 2024-04-26T04:46:48.406 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-04-26T04:46:48.551 INFO:teuthology.orchestra.run.smithi116.stdout:Last metadata expiration check: 0:01:26 ago on Fri 26 Apr 2024 04:45:22 AM UTC. 2024-04-26T04:46:48.669 INFO:teuthology.orchestra.run.smithi116.stdout:Dependencies resolved. 2024-04-26T04:46:48.670 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:46:48.670 INFO:teuthology.orchestra.run.smithi116.stdout: Package Arch Version Repository Size 2024-04-26T04:46:48.671 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:46:48.671 INFO:teuthology.orchestra.run.smithi116.stdout:Installing: 2024-04-26T04:46:48.671 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-fuse x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 812 k 2024-04-26T04:46:48.671 INFO:teuthology.orchestra.run.smithi116.stdout:Installing dependencies: 2024-04-26T04:46:48.671 INFO:teuthology.orchestra.run.smithi116.stdout: fuse x86_64 2.9.9-15.el9 baseos 80 k 2024-04-26T04:46:48.671 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:48.671 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction Summary 2024-04-26T04:46:48.671 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:46:48.671 INFO:teuthology.orchestra.run.smithi116.stdout:Install 2 Packages 2024-04-26T04:46:48.671 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:48.671 INFO:teuthology.orchestra.run.smithi116.stdout:Total download size: 892 k 2024-04-26T04:46:48.671 INFO:teuthology.orchestra.run.smithi116.stdout:Installed size: 2.6 M 2024-04-26T04:46:48.671 INFO:teuthology.orchestra.run.smithi116.stdout:Downloading Packages: 2024-04-26T04:46:48.954 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-04-26T04:46:49.043 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-04-26T04:46:49.105 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 5/6 2024-04-26T04:46:49.132 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 6/6 2024-04-26T04:46:49.272 INFO:teuthology.orchestra.run.smithi116.stdout:(1/2): ceph-fuse-17.2.7-904.gd8c6b0a3.el9.x86_6 1.5 MB/s | 812 kB 00:00 2024-04-26T04:46:49.663 INFO:teuthology.orchestra.run.smithi116.stdout:(2/2): fuse-2.9.9-15.el9.x86_64.rpm 87 kB/s | 80 kB 00:00 2024-04-26T04:46:49.665 INFO:teuthology.orchestra.run.smithi116.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:46:49.665 INFO:teuthology.orchestra.run.smithi116.stdout:Total 898 kB/s | 892 kB 00:00 2024-04-26T04:46:49.676 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction check 2024-04-26T04:46:49.699 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction check succeeded. 2024-04-26T04:46:49.700 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction test 2024-04-26T04:46:49.777 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction test succeeded. 2024-04-26T04:46:49.777 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction 2024-04-26T04:46:49.898 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 6/6 2024-04-26T04:46:49.898 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/6 2024-04-26T04:46:49.898 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-04-26T04:46:49.898 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 3/6 2024-04-26T04:46:49.898 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-04-26T04:46:49.899 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-04-26T04:46:49.968 INFO:teuthology.orchestra.run.smithi116.stdout: Preparing : 1/1 2024-04-26T04:46:50.109 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : fuse-2.9.9-15.el9.x86_64 1/2 2024-04-26T04:46:50.156 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 6/6 2024-04-26T04:46:50.156 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:50.156 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-04-26T04:46:50.157 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:46:50.157 INFO:teuthology.orchestra.run.smithi012.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-04-26T04:46:50.157 INFO:teuthology.orchestra.run.smithi012.stdout: python3-babel-2.9.1-2.el9.noarch 2024-04-26T04:46:50.157 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-04-26T04:46:50.157 INFO:teuthology.orchestra.run.smithi012.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-04-26T04:46:50.157 INFO:teuthology.orchestra.run.smithi012.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-04-26T04:46:50.157 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:50.157 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-04-26T04:46:50.177 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/2 2024-04-26T04:46:50.374 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-fuse 2024-04-26T04:46:50.859 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/2 2024-04-26T04:46:50.859 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/2 2024-04-26T04:46:50.911 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:27 ago on Fri 26 Apr 2024 04:45:23 AM UTC. 2024-04-26T04:46:51.037 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-04-26T04:46:51.038 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:46:51.038 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2024-04-26T04:46:51.038 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:46:51.038 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-04-26T04:46:51.038 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-fuse x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 812 k 2024-04-26T04:46:51.038 INFO:teuthology.orchestra.run.smithi012.stdout:Installing dependencies: 2024-04-26T04:46:51.038 INFO:teuthology.orchestra.run.smithi012.stdout: fuse x86_64 2.9.9-15.el9 baseos 80 k 2024-04-26T04:46:51.038 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:51.038 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-04-26T04:46:51.039 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:46:51.039 INFO:teuthology.orchestra.run.smithi012.stdout:Install 2 Packages 2024-04-26T04:46:51.039 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:51.039 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 892 k 2024-04-26T04:46:51.039 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 2.6 M 2024-04-26T04:46:51.039 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-04-26T04:46:51.114 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 2/2 2024-04-26T04:46:51.114 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:51.114 INFO:teuthology.orchestra.run.smithi116.stdout:Installed: 2024-04-26T04:46:51.114 INFO:teuthology.orchestra.run.smithi116.stdout: ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 fuse-2.9.9-15.el9.x86_64 2024-04-26T04:46:51.114 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:51.114 INFO:teuthology.orchestra.run.smithi116.stdout:Complete! 2024-04-26T04:46:51.367 DEBUG:teuthology.orchestra.run.smithi116:> sudo yum -y install librados-devel 2024-04-26T04:46:51.686 INFO:teuthology.orchestra.run.smithi012.stdout:(1/2): ceph-fuse-17.2.7-904.gd8c6b0a3.el9.x86_6 1.6 MB/s | 812 kB 00:00 2024-04-26T04:46:51.893 INFO:teuthology.orchestra.run.smithi116.stdout:Last metadata expiration check: 0:01:29 ago on Fri 26 Apr 2024 04:45:22 AM UTC. 2024-04-26T04:46:51.937 INFO:teuthology.orchestra.run.smithi012.stdout:(2/2): fuse-2.9.9-15.el9.x86_64.rpm 107 kB/s | 80 kB 00:00 2024-04-26T04:46:51.938 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:46:51.938 INFO:teuthology.orchestra.run.smithi012.stdout:Total 991 kB/s | 892 kB 00:00 2024-04-26T04:46:51.951 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-04-26T04:46:51.975 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-04-26T04:46:51.975 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-04-26T04:46:52.013 INFO:teuthology.orchestra.run.smithi116.stdout:Dependencies resolved. 2024-04-26T04:46:52.014 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:46:52.014 INFO:teuthology.orchestra.run.smithi116.stdout: Package Arch Version Repo Size 2024-04-26T04:46:52.014 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:46:52.014 INFO:teuthology.orchestra.run.smithi116.stdout:Installing: 2024-04-26T04:46:52.014 INFO:teuthology.orchestra.run.smithi116.stdout: librados-devel x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 126 k 2024-04-26T04:46:52.014 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:52.014 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction Summary 2024-04-26T04:46:52.015 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:46:52.015 INFO:teuthology.orchestra.run.smithi116.stdout:Install 1 Package 2024-04-26T04:46:52.015 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:52.015 INFO:teuthology.orchestra.run.smithi116.stdout:Total download size: 126 k 2024-04-26T04:46:52.015 INFO:teuthology.orchestra.run.smithi116.stdout:Installed size: 448 k 2024-04-26T04:46:52.015 INFO:teuthology.orchestra.run.smithi116.stdout:Downloading Packages: 2024-04-26T04:46:52.058 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-04-26T04:46:52.058 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-04-26T04:46:52.219 INFO:teuthology.orchestra.run.smithi116.stdout:librados-devel-17.2.7-904.gd8c6b0a3.el9.x86_64. 619 kB/s | 126 kB 00:00 2024-04-26T04:46:52.220 INFO:teuthology.orchestra.run.smithi116.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:46:52.220 INFO:teuthology.orchestra.run.smithi116.stdout:Total 615 kB/s | 126 kB 00:00 2024-04-26T04:46:52.220 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction check 2024-04-26T04:46:52.226 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction check succeeded. 2024-04-26T04:46:52.226 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction test 2024-04-26T04:46:52.242 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-04-26T04:46:52.287 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction test succeeded. 2024-04-26T04:46:52.287 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction 2024-04-26T04:46:52.377 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : fuse-2.9.9-15.el9.x86_64 1/2 2024-04-26T04:46:52.443 INFO:teuthology.orchestra.run.smithi116.stdout: Preparing : 1/1 2024-04-26T04:46:52.445 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/2 2024-04-26T04:46:52.565 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:46:52.966 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:46:53.175 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/2 2024-04-26T04:46:53.175 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/2 2024-04-26T04:46:53.191 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:46:53.191 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:53.191 INFO:teuthology.orchestra.run.smithi116.stdout:Installed: 2024-04-26T04:46:53.191 INFO:teuthology.orchestra.run.smithi116.stdout: librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:53.191 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:53.191 INFO:teuthology.orchestra.run.smithi116.stdout:Complete! 2024-04-26T04:46:53.447 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 2/2 2024-04-26T04:46:53.448 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:53.448 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-04-26T04:46:53.448 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 fuse-2.9.9-15.el9.x86_64 2024-04-26T04:46:53.448 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:53.448 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-04-26T04:46:53.453 DEBUG:teuthology.orchestra.run.smithi116:> sudo yum -y install libcephfs2 2024-04-26T04:46:53.683 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install librados-devel 2024-04-26T04:46:53.967 INFO:teuthology.orchestra.run.smithi116.stdout:Last metadata expiration check: 0:01:31 ago on Fri 26 Apr 2024 04:45:22 AM UTC. 2024-04-26T04:46:54.047 INFO:teuthology.orchestra.run.smithi116.stdout:Package libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:46:54.089 INFO:teuthology.orchestra.run.smithi116.stdout:Dependencies resolved. 2024-04-26T04:46:54.090 INFO:teuthology.orchestra.run.smithi116.stdout:Nothing to do. 2024-04-26T04:46:54.090 INFO:teuthology.orchestra.run.smithi116.stdout:Complete! 2024-04-26T04:46:54.206 DEBUG:teuthology.orchestra.run.smithi116:> sudo yum -y install libcephfs-devel 2024-04-26T04:46:54.215 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:31 ago on Fri 26 Apr 2024 04:45:23 AM UTC. 2024-04-26T04:46:54.340 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-04-26T04:46:54.341 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:46:54.341 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repo Size 2024-04-26T04:46:54.341 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:46:54.341 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-04-26T04:46:54.341 INFO:teuthology.orchestra.run.smithi012.stdout: librados-devel x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 126 k 2024-04-26T04:46:54.341 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:54.342 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-04-26T04:46:54.342 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:46:54.342 INFO:teuthology.orchestra.run.smithi012.stdout:Install 1 Package 2024-04-26T04:46:54.342 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:54.342 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 126 k 2024-04-26T04:46:54.342 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 448 k 2024-04-26T04:46:54.342 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-04-26T04:46:54.686 INFO:teuthology.orchestra.run.smithi012.stdout:librados-devel-17.2.7-904.gd8c6b0a3.el9.x86_64. 369 kB/s | 126 kB 00:00 2024-04-26T04:46:54.686 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:46:54.686 INFO:teuthology.orchestra.run.smithi012.stdout:Total 367 kB/s | 126 kB 00:00 2024-04-26T04:46:54.686 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-04-26T04:46:54.692 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-04-26T04:46:54.692 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-04-26T04:46:54.718 INFO:teuthology.orchestra.run.smithi116.stdout:Last metadata expiration check: 0:01:32 ago on Fri 26 Apr 2024 04:45:22 AM UTC. 2024-04-26T04:46:54.758 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-04-26T04:46:54.758 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-04-26T04:46:54.838 INFO:teuthology.orchestra.run.smithi116.stdout:Dependencies resolved. 2024-04-26T04:46:54.838 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:46:54.838 INFO:teuthology.orchestra.run.smithi116.stdout: Package Arch Version Repo Size 2024-04-26T04:46:54.839 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:46:54.839 INFO:teuthology.orchestra.run.smithi116.stdout:Installing: 2024-04-26T04:46:54.839 INFO:teuthology.orchestra.run.smithi116.stdout: libcephfs-devel x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 25 k 2024-04-26T04:46:54.839 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:54.839 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction Summary 2024-04-26T04:46:54.839 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:46:54.839 INFO:teuthology.orchestra.run.smithi116.stdout:Install 1 Package 2024-04-26T04:46:54.839 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:54.839 INFO:teuthology.orchestra.run.smithi116.stdout:Total download size: 25 k 2024-04-26T04:46:54.839 INFO:teuthology.orchestra.run.smithi116.stdout:Installed size: 110 k 2024-04-26T04:46:54.839 INFO:teuthology.orchestra.run.smithi116.stdout:Downloading Packages: 2024-04-26T04:46:54.916 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-04-26T04:46:55.007 INFO:teuthology.orchestra.run.smithi116.stdout:libcephfs-devel-17.2.7-904.gd8c6b0a3.el9.x86_64 149 kB/s | 25 kB 00:00 2024-04-26T04:46:55.008 INFO:teuthology.orchestra.run.smithi116.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:46:55.008 INFO:teuthology.orchestra.run.smithi116.stdout:Total 148 kB/s | 25 kB 00:00 2024-04-26T04:46:55.008 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction check 2024-04-26T04:46:55.011 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction check succeeded. 2024-04-26T04:46:55.011 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction test 2024-04-26T04:46:55.012 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction test succeeded. 2024-04-26T04:46:55.012 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction 2024-04-26T04:46:55.016 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:46:55.099 INFO:teuthology.orchestra.run.smithi116.stdout: Preparing : 1/1 2024-04-26T04:46:55.209 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:46:55.392 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:46:55.540 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:46:55.662 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:46:55.662 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:55.662 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-04-26T04:46:55.662 INFO:teuthology.orchestra.run.smithi012.stdout: librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:55.662 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:55.662 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-04-26T04:46:55.756 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:46:55.756 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:55.756 INFO:teuthology.orchestra.run.smithi116.stdout:Installed: 2024-04-26T04:46:55.756 INFO:teuthology.orchestra.run.smithi116.stdout: libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:55.756 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:46:55.756 INFO:teuthology.orchestra.run.smithi116.stdout:Complete! 2024-04-26T04:46:55.904 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install libcephfs2 2024-04-26T04:46:55.928 DEBUG:teuthology.orchestra.run.smithi116:> sudo yum -y install librados2 2024-04-26T04:46:56.439 INFO:teuthology.orchestra.run.smithi116.stdout:Last metadata expiration check: 0:01:34 ago on Fri 26 Apr 2024 04:45:22 AM UTC. 2024-04-26T04:46:56.457 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:33 ago on Fri 26 Apr 2024 04:45:23 AM UTC. 2024-04-26T04:46:56.517 INFO:teuthology.orchestra.run.smithi116.stdout:Package librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:46:56.540 INFO:teuthology.orchestra.run.smithi012.stdout:Package libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:46:56.557 INFO:teuthology.orchestra.run.smithi116.stdout:Dependencies resolved. 2024-04-26T04:46:56.558 INFO:teuthology.orchestra.run.smithi116.stdout:Nothing to do. 2024-04-26T04:46:56.558 INFO:teuthology.orchestra.run.smithi116.stdout:Complete! 2024-04-26T04:46:56.583 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-04-26T04:46:56.584 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2024-04-26T04:46:56.584 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-04-26T04:46:56.616 DEBUG:teuthology.orchestra.run.smithi116:> sudo yum -y install librbd1 2024-04-26T04:46:56.638 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install libcephfs-devel 2024-04-26T04:46:57.125 INFO:teuthology.orchestra.run.smithi116.stdout:Last metadata expiration check: 0:01:35 ago on Fri 26 Apr 2024 04:45:22 AM UTC. 2024-04-26T04:46:57.180 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:34 ago on Fri 26 Apr 2024 04:45:23 AM UTC. 2024-04-26T04:46:57.204 INFO:teuthology.orchestra.run.smithi116.stdout:Package librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:46:57.244 INFO:teuthology.orchestra.run.smithi116.stdout:Dependencies resolved. 2024-04-26T04:46:57.245 INFO:teuthology.orchestra.run.smithi116.stdout:Nothing to do. 2024-04-26T04:46:57.245 INFO:teuthology.orchestra.run.smithi116.stdout:Complete! 2024-04-26T04:46:57.304 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-04-26T04:46:57.305 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:46:57.305 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repo Size 2024-04-26T04:46:57.305 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:46:57.305 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-04-26T04:46:57.306 INFO:teuthology.orchestra.run.smithi012.stdout: libcephfs-devel x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 25 k 2024-04-26T04:46:57.306 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:57.306 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-04-26T04:46:57.306 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:46:57.306 INFO:teuthology.orchestra.run.smithi012.stdout:Install 1 Package 2024-04-26T04:46:57.306 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:57.306 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 25 k 2024-04-26T04:46:57.306 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 110 k 2024-04-26T04:46:57.306 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-04-26T04:46:57.309 DEBUG:teuthology.orchestra.run.smithi116:> sudo yum -y install python3-rados 2024-04-26T04:46:57.472 INFO:teuthology.orchestra.run.smithi012.stdout:libcephfs-devel-17.2.7-904.gd8c6b0a3.el9.x86_64 150 kB/s | 25 kB 00:00 2024-04-26T04:46:57.473 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:46:57.473 INFO:teuthology.orchestra.run.smithi012.stdout:Total 149 kB/s | 25 kB 00:00 2024-04-26T04:46:57.473 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-04-26T04:46:57.476 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-04-26T04:46:57.477 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-04-26T04:46:57.478 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-04-26T04:46:57.478 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-04-26T04:46:57.566 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-04-26T04:46:57.686 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:46:57.815 INFO:teuthology.orchestra.run.smithi116.stdout:Last metadata expiration check: 0:01:35 ago on Fri 26 Apr 2024 04:45:22 AM UTC. 2024-04-26T04:46:57.894 INFO:teuthology.orchestra.run.smithi116.stdout:Package python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:46:57.934 INFO:teuthology.orchestra.run.smithi116.stdout:Dependencies resolved. 2024-04-26T04:46:57.935 INFO:teuthology.orchestra.run.smithi116.stdout:Nothing to do. 2024-04-26T04:46:57.935 INFO:teuthology.orchestra.run.smithi116.stdout:Complete! 2024-04-26T04:46:57.997 DEBUG:teuthology.orchestra.run.smithi116:> sudo yum -y install python3-rgw 2024-04-26T04:46:58.048 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:46:58.254 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:46:58.254 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:58.254 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-04-26T04:46:58.254 INFO:teuthology.orchestra.run.smithi012.stdout: libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:46:58.254 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:46:58.254 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-04-26T04:46:58.421 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install librados2 2024-04-26T04:46:58.504 INFO:teuthology.orchestra.run.smithi116.stdout:Last metadata expiration check: 0:01:36 ago on Fri 26 Apr 2024 04:45:22 AM UTC. 2024-04-26T04:46:58.583 INFO:teuthology.orchestra.run.smithi116.stdout:Package python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:46:58.623 INFO:teuthology.orchestra.run.smithi116.stdout:Dependencies resolved. 2024-04-26T04:46:58.624 INFO:teuthology.orchestra.run.smithi116.stdout:Nothing to do. 2024-04-26T04:46:58.624 INFO:teuthology.orchestra.run.smithi116.stdout:Complete! 2024-04-26T04:46:58.691 DEBUG:teuthology.orchestra.run.smithi116:> sudo yum -y install python3-cephfs 2024-04-26T04:46:58.955 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:35 ago on Fri 26 Apr 2024 04:45:23 AM UTC. 2024-04-26T04:46:59.036 INFO:teuthology.orchestra.run.smithi012.stdout:Package librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:46:59.079 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-04-26T04:46:59.080 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2024-04-26T04:46:59.080 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-04-26T04:46:59.142 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install librbd1 2024-04-26T04:46:59.195 INFO:teuthology.orchestra.run.smithi116.stdout:Last metadata expiration check: 0:01:37 ago on Fri 26 Apr 2024 04:45:22 AM UTC. 2024-04-26T04:46:59.273 INFO:teuthology.orchestra.run.smithi116.stdout:Package python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:46:59.314 INFO:teuthology.orchestra.run.smithi116.stdout:Dependencies resolved. 2024-04-26T04:46:59.315 INFO:teuthology.orchestra.run.smithi116.stdout:Nothing to do. 2024-04-26T04:46:59.315 INFO:teuthology.orchestra.run.smithi116.stdout:Complete! 2024-04-26T04:46:59.382 DEBUG:teuthology.orchestra.run.smithi116:> sudo yum -y install python3-rbd 2024-04-26T04:46:59.704 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:36 ago on Fri 26 Apr 2024 04:45:23 AM UTC. 2024-04-26T04:46:59.787 INFO:teuthology.orchestra.run.smithi012.stdout:Package librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:46:59.830 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-04-26T04:46:59.831 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2024-04-26T04:46:59.831 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-04-26T04:46:59.888 INFO:teuthology.orchestra.run.smithi116.stdout:Last metadata expiration check: 0:01:37 ago on Fri 26 Apr 2024 04:45:22 AM UTC. 2024-04-26T04:46:59.898 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install python3-rados 2024-04-26T04:46:59.966 INFO:teuthology.orchestra.run.smithi116.stdout:Package python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:47:00.006 INFO:teuthology.orchestra.run.smithi116.stdout:Dependencies resolved. 2024-04-26T04:47:00.007 INFO:teuthology.orchestra.run.smithi116.stdout:Nothing to do. 2024-04-26T04:47:00.007 INFO:teuthology.orchestra.run.smithi116.stdout:Complete! 2024-04-26T04:47:00.075 DEBUG:teuthology.orchestra.run.smithi116:> sudo yum -y install rbd-fuse 2024-04-26T04:47:00.428 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:37 ago on Fri 26 Apr 2024 04:45:23 AM UTC. 2024-04-26T04:47:00.510 INFO:teuthology.orchestra.run.smithi012.stdout:Package python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:47:00.552 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-04-26T04:47:00.553 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2024-04-26T04:47:00.553 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-04-26T04:47:00.583 INFO:teuthology.orchestra.run.smithi116.stdout:Last metadata expiration check: 0:01:38 ago on Fri 26 Apr 2024 04:45:22 AM UTC. 2024-04-26T04:47:00.617 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install python3-rgw 2024-04-26T04:47:00.701 INFO:teuthology.orchestra.run.smithi116.stdout:Dependencies resolved. 2024-04-26T04:47:00.702 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:47:00.702 INFO:teuthology.orchestra.run.smithi116.stdout: Package Architecture Version Repository Size 2024-04-26T04:47:00.702 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:47:00.702 INFO:teuthology.orchestra.run.smithi116.stdout:Installing: 2024-04-26T04:47:00.702 INFO:teuthology.orchestra.run.smithi116.stdout: rbd-fuse x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 82 k 2024-04-26T04:47:00.702 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:47:00.702 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction Summary 2024-04-26T04:47:00.702 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:47:00.702 INFO:teuthology.orchestra.run.smithi116.stdout:Install 1 Package 2024-04-26T04:47:00.703 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:47:00.703 INFO:teuthology.orchestra.run.smithi116.stdout:Total download size: 82 k 2024-04-26T04:47:00.703 INFO:teuthology.orchestra.run.smithi116.stdout:Installed size: 222 k 2024-04-26T04:47:00.703 INFO:teuthology.orchestra.run.smithi116.stdout:Downloading Packages: 2024-04-26T04:47:00.918 INFO:teuthology.orchestra.run.smithi116.stdout:rbd-fuse-17.2.7-904.gd8c6b0a3.el9.x86_64.rpm 380 kB/s | 82 kB 00:00 2024-04-26T04:47:00.919 INFO:teuthology.orchestra.run.smithi116.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:47:00.919 INFO:teuthology.orchestra.run.smithi116.stdout:Total 378 kB/s | 82 kB 00:00 2024-04-26T04:47:00.919 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction check 2024-04-26T04:47:00.925 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction check succeeded. 2024-04-26T04:47:00.925 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction test 2024-04-26T04:47:00.987 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction test succeeded. 2024-04-26T04:47:00.988 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction 2024-04-26T04:47:01.129 INFO:teuthology.orchestra.run.smithi116.stdout: Preparing : 1/1 2024-04-26T04:47:01.150 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:38 ago on Fri 26 Apr 2024 04:45:23 AM UTC. 2024-04-26T04:47:01.230 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:01.232 INFO:teuthology.orchestra.run.smithi012.stdout:Package python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:47:01.275 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-04-26T04:47:01.276 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2024-04-26T04:47:01.276 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-04-26T04:47:01.342 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install python3-cephfs 2024-04-26T04:47:01.575 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:01.812 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:01.812 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:47:01.812 INFO:teuthology.orchestra.run.smithi116.stdout:Installed: 2024-04-26T04:47:01.812 INFO:teuthology.orchestra.run.smithi116.stdout: rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:47:01.812 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:47:01.812 INFO:teuthology.orchestra.run.smithi116.stdout:Complete! 2024-04-26T04:47:01.873 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:38 ago on Fri 26 Apr 2024 04:45:23 AM UTC. 2024-04-26T04:47:01.953 INFO:teuthology.orchestra.run.smithi012.stdout:Package python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:47:01.996 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-04-26T04:47:01.997 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2024-04-26T04:47:01.998 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-04-26T04:47:02.063 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install python3-rbd 2024-04-26T04:47:02.068 DEBUG:teuthology.orchestra.run.smithi116:> sudo yum -y install rbd-mirror 2024-04-26T04:47:02.578 INFO:teuthology.orchestra.run.smithi116.stdout:Last metadata expiration check: 0:01:40 ago on Fri 26 Apr 2024 04:45:22 AM UTC. 2024-04-26T04:47:02.601 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:39 ago on Fri 26 Apr 2024 04:45:23 AM UTC. 2024-04-26T04:47:02.682 INFO:teuthology.orchestra.run.smithi012.stdout:Package python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:47:02.699 INFO:teuthology.orchestra.run.smithi116.stdout:Dependencies resolved. 2024-04-26T04:47:02.699 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:47:02.699 INFO:teuthology.orchestra.run.smithi116.stdout: Package Arch Version Repo Size 2024-04-26T04:47:02.699 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:47:02.699 INFO:teuthology.orchestra.run.smithi116.stdout:Installing: 2024-04-26T04:47:02.699 INFO:teuthology.orchestra.run.smithi116.stdout: rbd-mirror x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 2.9 M 2024-04-26T04:47:02.699 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:47:02.699 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction Summary 2024-04-26T04:47:02.699 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:47:02.699 INFO:teuthology.orchestra.run.smithi116.stdout:Install 1 Package 2024-04-26T04:47:02.699 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:47:02.700 INFO:teuthology.orchestra.run.smithi116.stdout:Total download size: 2.9 M 2024-04-26T04:47:02.700 INFO:teuthology.orchestra.run.smithi116.stdout:Installed size: 12 M 2024-04-26T04:47:02.700 INFO:teuthology.orchestra.run.smithi116.stdout:Downloading Packages: 2024-04-26T04:47:02.725 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-04-26T04:47:02.726 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2024-04-26T04:47:02.726 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-04-26T04:47:02.788 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install rbd-fuse 2024-04-26T04:47:03.326 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:40 ago on Fri 26 Apr 2024 04:45:23 AM UTC. 2024-04-26T04:47:03.451 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-04-26T04:47:03.452 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:47:03.452 INFO:teuthology.orchestra.run.smithi012.stdout: Package Architecture Version Repository Size 2024-04-26T04:47:03.452 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:47:03.452 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-04-26T04:47:03.452 INFO:teuthology.orchestra.run.smithi012.stdout: rbd-fuse x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 82 k 2024-04-26T04:47:03.452 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:47:03.452 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-04-26T04:47:03.452 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:47:03.452 INFO:teuthology.orchestra.run.smithi012.stdout:Install 1 Package 2024-04-26T04:47:03.452 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:47:03.453 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 82 k 2024-04-26T04:47:03.453 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 222 k 2024-04-26T04:47:03.453 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-04-26T04:47:03.658 INFO:teuthology.orchestra.run.smithi012.stdout:rbd-fuse-17.2.7-904.gd8c6b0a3.el9.x86_64.rpm 398 kB/s | 82 kB 00:00 2024-04-26T04:47:03.659 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:47:03.659 INFO:teuthology.orchestra.run.smithi012.stdout:Total 396 kB/s | 82 kB 00:00 2024-04-26T04:47:03.659 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-04-26T04:47:03.666 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-04-26T04:47:03.667 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-04-26T04:47:03.732 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-04-26T04:47:03.732 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-04-26T04:47:03.862 INFO:teuthology.orchestra.run.smithi116.stdout:rbd-mirror-17.2.7-904.gd8c6b0a3.el9.x86_64.rpm 2.5 MB/s | 2.9 MB 00:01 2024-04-26T04:47:03.863 INFO:teuthology.orchestra.run.smithi116.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:47:03.864 INFO:teuthology.orchestra.run.smithi116.stdout:Total 2.5 MB/s | 2.9 MB 00:01 2024-04-26T04:47:03.864 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction check 2024-04-26T04:47:03.872 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction check succeeded. 2024-04-26T04:47:03.873 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction test 2024-04-26T04:47:03.883 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-04-26T04:47:03.950 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction test succeeded. 2024-04-26T04:47:03.951 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction 2024-04-26T04:47:03.972 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:04.327 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:04.361 INFO:teuthology.orchestra.run.smithi116.stdout: Preparing : 1/1 2024-04-26T04:47:04.438 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:04.462 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:04.463 INFO:teuthology.orchestra.run.smithi116.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:47:04.463 INFO:teuthology.orchestra.run.smithi116.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-04-26T04:47:04.463 INFO:teuthology.orchestra.run.smithi116.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-26T04:47:04.463 INFO:teuthology.orchestra.run.smithi116.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-26T04:47:04.463 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:47:04.567 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:04.567 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:47:04.567 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-04-26T04:47:04.568 INFO:teuthology.orchestra.run.smithi012.stdout: rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:47:04.568 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:47:04.568 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-04-26T04:47:04.801 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install rbd-mirror 2024-04-26T04:47:05.334 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:42 ago on Fri 26 Apr 2024 04:45:23 AM UTC. 2024-04-26T04:47:05.459 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-04-26T04:47:05.460 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:47:05.460 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repo Size 2024-04-26T04:47:05.460 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:47:05.460 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-04-26T04:47:05.460 INFO:teuthology.orchestra.run.smithi012.stdout: rbd-mirror x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 2.9 M 2024-04-26T04:47:05.461 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:47:05.461 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-04-26T04:47:05.461 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:47:05.461 INFO:teuthology.orchestra.run.smithi012.stdout:Install 1 Package 2024-04-26T04:47:05.461 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:47:05.461 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 2.9 M 2024-04-26T04:47:05.461 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 12 M 2024-04-26T04:47:05.461 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-04-26T04:47:05.606 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:05.606 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:47:05.606 INFO:teuthology.orchestra.run.smithi116.stdout:Installed: 2024-04-26T04:47:05.606 INFO:teuthology.orchestra.run.smithi116.stdout: rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:47:05.606 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:47:05.607 INFO:teuthology.orchestra.run.smithi116.stdout:Complete! 2024-04-26T04:47:05.903 DEBUG:teuthology.orchestra.run.smithi116:> sudo yum -y install rbd-nbd 2024-04-26T04:47:06.427 INFO:teuthology.orchestra.run.smithi116.stdout:Last metadata expiration check: 0:01:44 ago on Fri 26 Apr 2024 04:45:22 AM UTC. 2024-04-26T04:47:06.487 INFO:teuthology.orchestra.run.smithi012.stdout:rbd-mirror-17.2.7-904.gd8c6b0a3.el9.x86_64.rpm 2.9 MB/s | 2.9 MB 00:01 2024-04-26T04:47:06.487 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:47:06.488 INFO:teuthology.orchestra.run.smithi012.stdout:Total 2.9 MB/s | 2.9 MB 00:01 2024-04-26T04:47:06.488 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-04-26T04:47:06.500 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-04-26T04:47:06.500 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-04-26T04:47:06.551 INFO:teuthology.orchestra.run.smithi116.stdout:Dependencies resolved. 2024-04-26T04:47:07.118 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-04-26T04:47:07.118 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-04-26T04:47:07.118 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-04-26T04:47:07.118 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:07.118 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:07.118 INFO:teuthology.orchestra.run.smithi012.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:47:07.118 INFO:teuthology.orchestra.run.smithi012.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-04-26T04:47:07.118 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-26T04:47:07.118 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-26T04:47:07.118 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:47:07.119 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:47:07.119 INFO:teuthology.orchestra.run.smithi116.stdout: Package Architecture Version Repository Size 2024-04-26T04:47:07.120 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:47:07.120 INFO:teuthology.orchestra.run.smithi116.stdout:Installing: 2024-04-26T04:47:07.120 INFO:teuthology.orchestra.run.smithi116.stdout: rbd-nbd x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 165 k 2024-04-26T04:47:07.120 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:47:07.120 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction Summary 2024-04-26T04:47:07.120 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-04-26T04:47:07.120 INFO:teuthology.orchestra.run.smithi116.stdout:Install 1 Package 2024-04-26T04:47:07.120 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:47:07.121 INFO:teuthology.orchestra.run.smithi116.stdout:Total download size: 165 k 2024-04-26T04:47:07.121 INFO:teuthology.orchestra.run.smithi116.stdout:Installed size: 477 k 2024-04-26T04:47:07.121 INFO:teuthology.orchestra.run.smithi116.stdout:Downloading Packages: 2024-04-26T04:47:07.122 INFO:teuthology.orchestra.run.smithi116.stdout:rbd-nbd-17.2.7-904.gd8c6b0a3.el9.x86_64.rpm 713 kB/s | 165 kB 00:00 2024-04-26T04:47:07.122 INFO:teuthology.orchestra.run.smithi116.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:47:07.122 INFO:teuthology.orchestra.run.smithi116.stdout:Total 708 kB/s | 165 kB 00:00 2024-04-26T04:47:07.122 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction check 2024-04-26T04:47:07.122 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction check succeeded. 2024-04-26T04:47:07.122 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction test 2024-04-26T04:47:07.122 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction test succeeded. 2024-04-26T04:47:07.122 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction 2024-04-26T04:47:07.123 INFO:teuthology.orchestra.run.smithi116.stdout: Preparing : 1/1 2024-04-26T04:47:07.123 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:07.506 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:07.737 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:07.738 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:47:07.738 INFO:teuthology.orchestra.run.smithi116.stdout:Installed: 2024-04-26T04:47:07.738 INFO:teuthology.orchestra.run.smithi116.stdout: rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:47:07.738 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:47:07.738 INFO:teuthology.orchestra.run.smithi116.stdout:Complete! 2024-04-26T04:47:07.987 DEBUG:teuthology.parallel:result is None 2024-04-26T04:47:08.118 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:08.118 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:47:08.118 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-04-26T04:47:08.118 INFO:teuthology.orchestra.run.smithi012.stdout: rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:47:08.118 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:47:08.118 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-04-26T04:47:08.345 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install rbd-nbd 2024-04-26T04:47:08.885 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:45 ago on Fri 26 Apr 2024 04:45:23 AM UTC. 2024-04-26T04:47:09.009 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-04-26T04:47:09.010 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:47:09.010 INFO:teuthology.orchestra.run.smithi012.stdout: Package Architecture Version Repository Size 2024-04-26T04:47:09.010 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:47:09.010 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-04-26T04:47:09.010 INFO:teuthology.orchestra.run.smithi012.stdout: rbd-nbd x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 165 k 2024-04-26T04:47:09.010 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:47:09.011 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-04-26T04:47:09.011 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-04-26T04:47:09.011 INFO:teuthology.orchestra.run.smithi012.stdout:Install 1 Package 2024-04-26T04:47:09.011 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:47:09.011 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 165 k 2024-04-26T04:47:09.011 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 477 k 2024-04-26T04:47:09.011 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-04-26T04:47:09.232 INFO:teuthology.orchestra.run.smithi012.stdout:rbd-nbd-17.2.7-904.gd8c6b0a3.el9.x86_64.rpm 745 kB/s | 165 kB 00:00 2024-04-26T04:47:09.233 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:47:09.233 INFO:teuthology.orchestra.run.smithi012.stdout:Total 740 kB/s | 165 kB 00:00 2024-04-26T04:47:09.233 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-04-26T04:47:09.241 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-04-26T04:47:09.241 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-04-26T04:47:09.307 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-04-26T04:47:09.308 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-04-26T04:47:09.467 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-04-26T04:47:09.565 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:09.958 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:10.183 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:47:10.183 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:47:10.183 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-04-26T04:47:10.183 INFO:teuthology.orchestra.run.smithi012.stdout: rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:47:10.183 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:47:10.183 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-04-26T04:47:10.426 DEBUG:teuthology.parallel:result is None 2024-04-26T04:47:10.427 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T04:47:10.427 INFO:teuthology.packaging:ref: None 2024-04-26T04:47:10.427 INFO:teuthology.packaging:tag: None 2024-04-26T04:47:10.427 INFO:teuthology.packaging:branch: quincy 2024-04-26T04:47:10.427 INFO:teuthology.packaging:sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:47:10.427 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-26T04:47:10.643 DEBUG:teuthology.orchestra.run.smithi012:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-04-26T04:47:10.665 INFO:teuthology.orchestra.run.smithi012.stdout:17.2.7-904.gd8c6b0a3.el9 2024-04-26T04:47:10.666 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-904.gd8c6b0a3.el9 2024-04-26T04:47:10.666 INFO:teuthology.task.install:The correct ceph version 17.2.7-904.gd8c6b0a3 is installed. 2024-04-26T04:47:10.667 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T04:47:10.667 INFO:teuthology.packaging:ref: None 2024-04-26T04:47:10.667 INFO:teuthology.packaging:tag: None 2024-04-26T04:47:10.668 INFO:teuthology.packaging:branch: quincy 2024-04-26T04:47:10.668 INFO:teuthology.packaging:sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:47:10.668 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-26T04:47:10.879 DEBUG:teuthology.orchestra.run.smithi116:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-04-26T04:47:10.904 INFO:teuthology.orchestra.run.smithi116.stdout:17.2.7-904.gd8c6b0a3.el9 2024-04-26T04:47:10.904 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-904.gd8c6b0a3.el9 2024-04-26T04:47:10.905 INFO:teuthology.task.install:The correct ceph version 17.2.7-904.gd8c6b0a3 is installed. 2024-04-26T04:47:10.907 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-04-26T04:47:10.907 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-04-26T04:47:10.907 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-04-26T04:47:10.934 DEBUG:teuthology.orchestra.run.smithi116:> set -ex 2024-04-26T04:47:10.934 DEBUG:teuthology.orchestra.run.smithi116:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-04-26T04:47:10.973 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-04-26T04:47:10.974 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-04-26T04:47:10.974 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/usr/bin/daemon-helper 2024-04-26T04:47:10.999 DEBUG:teuthology.orchestra.run.smithi012:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-04-26T04:47:11.065 DEBUG:teuthology.orchestra.run.smithi116:> set -ex 2024-04-26T04:47:11.065 DEBUG:teuthology.orchestra.run.smithi116:> sudo dd of=/usr/bin/daemon-helper 2024-04-26T04:47:11.091 DEBUG:teuthology.orchestra.run.smithi116:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-04-26T04:47:11.157 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-04-26T04:47:11.158 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-04-26T04:47:11.158 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/usr/bin/adjust-ulimits 2024-04-26T04:47:11.196 DEBUG:teuthology.orchestra.run.smithi012:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-04-26T04:47:11.261 DEBUG:teuthology.orchestra.run.smithi116:> set -ex 2024-04-26T04:47:11.261 DEBUG:teuthology.orchestra.run.smithi116:> sudo dd of=/usr/bin/adjust-ulimits 2024-04-26T04:47:11.285 DEBUG:teuthology.orchestra.run.smithi116:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-04-26T04:47:11.350 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-04-26T04:47:11.350 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-04-26T04:47:11.350 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/usr/bin/stdin-killer 2024-04-26T04:47:11.375 DEBUG:teuthology.orchestra.run.smithi012:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-04-26T04:47:11.440 DEBUG:teuthology.orchestra.run.smithi116:> set -ex 2024-04-26T04:47:11.440 DEBUG:teuthology.orchestra.run.smithi116:> sudo dd of=/usr/bin/stdin-killer 2024-04-26T04:47:11.464 DEBUG:teuthology.orchestra.run.smithi116:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-04-26T04:47:11.529 INFO:teuthology.run_tasks:Running task cephadm... 2024-04-26T04:47:11.640 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'b22e2ebdeb24376882b7bda2a7329c8cccc2276a'} 2024-04-26T04:47:11.641 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2024-04-26T04:47:11.641 INFO:tasks.cephadm:Cluster fsid is 0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T04:47:11.641 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-04-26T04:47:11.641 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.12', 'mon.c': '[v2:172.21.15.12:3301,v1:172.21.15.12:6790]', 'mon.b': '172.21.15.116'} 2024-04-26T04:47:11.641 INFO:tasks.cephadm:First mon is mon.a on smithi012 2024-04-26T04:47:11.642 INFO:tasks.cephadm:First mgr is y 2024-04-26T04:47:11.642 INFO:tasks.cephadm:Normalizing hostnames... 2024-04-26T04:47:11.642 DEBUG:teuthology.orchestra.run.smithi012:> sudo hostname $(hostname -s) 2024-04-26T04:47:11.668 DEBUG:teuthology.orchestra.run.smithi116:> sudo hostname $(hostname -s) 2024-04-26T04:47:11.692 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2024-04-26T04:47:11.692 DEBUG:teuthology.orchestra.run.smithi012:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-26T04:47:11.935 INFO:teuthology.orchestra.run.smithi012.stdout:-rw-r--r--. 1 ubuntu ubuntu 375153 Apr 26 04:47 /home/ubuntu/cephtest/cephadm 2024-04-26T04:47:11.936 DEBUG:teuthology.orchestra.run.smithi116:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-26T04:47:12.170 INFO:teuthology.orchestra.run.smithi116.stdout:-rw-r--r--. 1 ubuntu ubuntu 375153 Apr 26 04:47 /home/ubuntu/cephtest/cephadm 2024-04-26T04:47:12.170 DEBUG:teuthology.orchestra.run.smithi012:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-26T04:47:12.188 DEBUG:teuthology.orchestra.run.smithi116:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-26T04:47:12.219 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2024-04-26T04:47:12.219 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-04-26T04:47:12.231 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-04-26T04:47:13.779 INFO:teuthology.orchestra.run.smithi012.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-26T04:47:13.780 INFO:teuthology.orchestra.run.smithi116.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-26T04:47:41.021 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-04-26T04:47:41.021 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph_version": "ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)", 2024-04-26T04:47:41.022 INFO:teuthology.orchestra.run.smithi116.stdout: "image_id": "0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43", 2024-04-26T04:47:41.022 INFO:teuthology.orchestra.run.smithi116.stdout: "repo_digests": [ 2024-04-26T04:47:41.022 INFO:teuthology.orchestra.run.smithi116.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17" 2024-04-26T04:47:41.022 INFO:teuthology.orchestra.run.smithi116.stdout: ] 2024-04-26T04:47:41.022 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-04-26T04:48:09.125 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-04-26T04:48:09.125 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph_version": "ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)", 2024-04-26T04:48:09.125 INFO:teuthology.orchestra.run.smithi012.stdout: "image_id": "0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43", 2024-04-26T04:48:09.125 INFO:teuthology.orchestra.run.smithi012.stdout: "repo_digests": [ 2024-04-26T04:48:09.125 INFO:teuthology.orchestra.run.smithi012.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17" 2024-04-26T04:48:09.126 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-04-26T04:48:09.126 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-04-26T04:48:09.146 DEBUG:teuthology.orchestra.run.smithi012:> sudo mkdir -p /etc/ceph 2024-04-26T04:48:09.193 DEBUG:teuthology.orchestra.run.smithi116:> sudo mkdir -p /etc/ceph 2024-04-26T04:48:09.225 DEBUG:teuthology.orchestra.run.smithi012:> sudo chmod 777 /etc/ceph 2024-04-26T04:48:09.249 DEBUG:teuthology.orchestra.run.smithi116:> sudo chmod 777 /etc/ceph 2024-04-26T04:48:09.292 INFO:tasks.cephadm:Writing seed config... 2024-04-26T04:48:09.293 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-04-26T04:48:09.293 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-04-26T04:48:09.293 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-04-26T04:48:09.293 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-04-26T04:48:09.293 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-04-26T04:48:09.294 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2024-04-26T04:48:09.294 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-04-26T04:48:09.294 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-04-26T04:48:09.294 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-04-26T04:48:09.294 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-04-26T04:48:09.294 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-04-26T04:48:09.294 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-04-26T04:48:09.295 DEBUG:teuthology.orchestra.run.smithi012:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-04-26T04:48:09.310 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 0b9350c6-0388-11ef-bc93-c7b262605968 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-04-26T04:48:09.310 DEBUG:teuthology.orchestra.run.smithi012:mon.a> sudo journalctl -f -n 0 -u ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mon.a.service 2024-04-26T04:48:09.352 DEBUG:teuthology.orchestra.run.smithi012:mgr.y> sudo journalctl -f -n 0 -u ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mgr.y.service 2024-04-26T04:48:09.395 INFO:tasks.cephadm:Bootstrapping... 2024-04-26T04:48:09.396 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.12 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-04-26T04:48:09.605 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:48:09.605 INFO:teuthology.orchestra.run.smithi012.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', '0b9350c6-0388-11ef-bc93-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.12', '--skip-admin-label'] 2024-04-26T04:48:09.631 INFO:teuthology.orchestra.run.smithi012.stdout:/bin/podman: stdout 4.9.4-dev 2024-04-26T04:48:09.632 INFO:teuthology.orchestra.run.smithi012.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-04-26T04:48:09.632 INFO:teuthology.orchestra.run.smithi012.stdout:Verifying podman|docker is present... 2024-04-26T04:48:09.656 INFO:teuthology.orchestra.run.smithi012.stdout:/bin/podman: stdout 4.9.4-dev 2024-04-26T04:48:09.656 INFO:teuthology.orchestra.run.smithi012.stdout:Verifying lvm2 is present... 2024-04-26T04:48:09.656 INFO:teuthology.orchestra.run.smithi012.stdout:Verifying time synchronization is in place... 2024-04-26T04:48:09.666 INFO:teuthology.orchestra.run.smithi012.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-04-26T04:48:09.667 INFO:teuthology.orchestra.run.smithi012.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-04-26T04:48:09.670 INFO:teuthology.orchestra.run.smithi012.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-04-26T04:48:09.670 INFO:teuthology.orchestra.run.smithi012.stdout:systemctl: stdout inactive 2024-04-26T04:48:09.677 INFO:teuthology.orchestra.run.smithi012.stdout:systemctl: stdout enabled 2024-04-26T04:48:09.683 INFO:teuthology.orchestra.run.smithi012.stdout:systemctl: stdout active 2024-04-26T04:48:09.684 INFO:teuthology.orchestra.run.smithi012.stdout:Unit chronyd.service is enabled and running 2024-04-26T04:48:09.684 INFO:teuthology.orchestra.run.smithi012.stdout:Repeating the final host check... 2024-04-26T04:48:09.707 INFO:teuthology.orchestra.run.smithi012.stdout:/bin/podman: stdout 4.9.4-dev 2024-04-26T04:48:09.708 INFO:teuthology.orchestra.run.smithi012.stdout:podman (/bin/podman) version 4.9.4 is present 2024-04-26T04:48:09.708 INFO:teuthology.orchestra.run.smithi012.stdout:systemctl is present 2024-04-26T04:48:09.708 INFO:teuthology.orchestra.run.smithi012.stdout:lvcreate is present 2024-04-26T04:48:09.714 INFO:teuthology.orchestra.run.smithi012.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-04-26T04:48:09.714 INFO:teuthology.orchestra.run.smithi012.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-04-26T04:48:09.721 INFO:teuthology.orchestra.run.smithi012.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-04-26T04:48:09.721 INFO:teuthology.orchestra.run.smithi012.stdout:systemctl: stdout inactive 2024-04-26T04:48:09.728 INFO:teuthology.orchestra.run.smithi012.stdout:systemctl: stdout enabled 2024-04-26T04:48:09.736 INFO:teuthology.orchestra.run.smithi012.stdout:systemctl: stdout active 2024-04-26T04:48:09.736 INFO:teuthology.orchestra.run.smithi012.stdout:Unit chronyd.service is enabled and running 2024-04-26T04:48:09.736 INFO:teuthology.orchestra.run.smithi012.stdout:Host looks OK 2024-04-26T04:48:09.737 INFO:teuthology.orchestra.run.smithi012.stdout:Cluster fsid: 0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T04:48:09.737 INFO:teuthology.orchestra.run.smithi012.stdout:Acquiring lock 140211427705040 on /run/cephadm/0b9350c6-0388-11ef-bc93-c7b262605968.lock 2024-04-26T04:48:09.737 INFO:teuthology.orchestra.run.smithi012.stdout:Lock 140211427705040 acquired on /run/cephadm/0b9350c6-0388-11ef-bc93-c7b262605968.lock 2024-04-26T04:48:09.737 INFO:teuthology.orchestra.run.smithi012.stdout:Verifying IP 172.21.15.12 port 3300 ... 2024-04-26T04:48:09.737 INFO:teuthology.orchestra.run.smithi012.stdout:Verifying IP 172.21.15.12 port 6789 ... 2024-04-26T04:48:09.737 INFO:teuthology.orchestra.run.smithi012.stdout:Base mon IP(s) is [172.21.15.12:3300, 172.21.15.12:6789], mon addrv is [v2:172.21.15.12:3300,v1:172.21.15.12:6789] 2024-04-26T04:48:09.739 INFO:teuthology.orchestra.run.smithi012.stdout:/sbin/ip: stdout default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.12 metric 100 2024-04-26T04:48:09.739 INFO:teuthology.orchestra.run.smithi012.stdout:/sbin/ip: stdout 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.12 metric 100 2024-04-26T04:48:09.742 INFO:teuthology.orchestra.run.smithi012.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-04-26T04:48:09.742 INFO:teuthology.orchestra.run.smithi012.stdout:/sbin/ip: stdout fe80::/64 dev ens1f0 proto kernel metric 1024 pref medium 2024-04-26T04:48:09.742 INFO:teuthology.orchestra.run.smithi012.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 100 pref medium 2024-04-26T04:48:09.745 INFO:teuthology.orchestra.run.smithi012.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-04-26T04:48:09.745 INFO:teuthology.orchestra.run.smithi012.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2024-04-26T04:48:09.745 INFO:teuthology.orchestra.run.smithi012.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-04-26T04:48:09.745 INFO:teuthology.orchestra.run.smithi012.stdout:/sbin/ip: stdout 4: ens1f0: mtu 1500 state UP qlen 1000 2024-04-26T04:48:09.745 INFO:teuthology.orchestra.run.smithi012.stdout:/sbin/ip: stdout inet6 fe80::ae1f:6bff:fef8:25ea/64 scope link noprefixroute 2024-04-26T04:48:09.745 INFO:teuthology.orchestra.run.smithi012.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-04-26T04:48:09.745 INFO:teuthology.orchestra.run.smithi012.stdout:Mon IP `172.21.15.12` is in CIDR network `172.21.0.0/20` 2024-04-26T04:48:09.745 INFO:teuthology.orchestra.run.smithi012.stdout:Mon IP `172.21.15.12` is in CIDR network `172.21.0.0/20` 2024-04-26T04:48:09.745 INFO:teuthology.orchestra.run.smithi012.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2024-04-26T04:48:09.746 INFO:teuthology.orchestra.run.smithi012.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-04-26T04:48:09.746 INFO:teuthology.orchestra.run.smithi012.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-26T04:48:10.234 INFO:teuthology.orchestra.run.smithi012.stdout:/bin/podman: stdout 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 2024-04-26T04:48:10.234 INFO:teuthology.orchestra.run.smithi012.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-26T04:48:10.234 INFO:teuthology.orchestra.run.smithi012.stdout:/bin/podman: stderr Getting image source signatures 2024-04-26T04:48:10.234 INFO:teuthology.orchestra.run.smithi012.stdout:/bin/podman: stderr Copying blob sha256:0e2af62137028b073c6ccdb8d4d7e0ccccc475ce5f6e225cb983e90813813045 2024-04-26T04:48:10.234 INFO:teuthology.orchestra.run.smithi012.stdout:/bin/podman: stderr Copying blob sha256:3d665f4c032644be9cae748af04bee69f2bfe96cdc3616dabe0ab17899633ab5 2024-04-26T04:48:10.234 INFO:teuthology.orchestra.run.smithi012.stdout:/bin/podman: stderr Copying config sha256:0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 2024-04-26T04:48:10.234 INFO:teuthology.orchestra.run.smithi012.stdout:/bin/podman: stderr Writing manifest to image destination 2024-04-26T04:48:11.182 INFO:teuthology.orchestra.run.smithi012.stdout:ceph: stdout ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable) 2024-04-26T04:48:11.183 INFO:teuthology.orchestra.run.smithi012.stdout:Ceph version: ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable) 2024-04-26T04:48:11.183 INFO:teuthology.orchestra.run.smithi012.stdout:Extracting ceph user uid/gid from container image... 2024-04-26T04:48:12.246 INFO:teuthology.orchestra.run.smithi012.stdout:stat: stdout 167 167 2024-04-26T04:48:12.247 INFO:teuthology.orchestra.run.smithi012.stdout:Creating initial keys... 2024-04-26T04:48:13.128 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph-authtool: stdout AQAMMitm5AIDIhAADBvgkdcrryJUOYO/ZZdgKQ== 2024-04-26T04:48:13.992 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph-authtool: stdout AQANMitm6T/nHRAAhf54be8V5jLzqcRFaFdcig== 2024-04-26T04:48:14.884 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph-authtool: stdout AQAOMitmx/fgEhAA0V5BQn7o5aFSt1niFNcr1g== 2024-04-26T04:48:14.884 INFO:teuthology.orchestra.run.smithi012.stdout:Creating initial monmap... 2024-04-26T04:48:15.754 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-26T04:48:15.754 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2024-04-26T04:48:15.755 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to 0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T04:48:15.755 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-26T04:48:15.755 INFO:teuthology.orchestra.run.smithi012.stdout:monmaptool for a [v2:172.21.15.12:3300,v1:172.21.15.12:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-26T04:48:15.755 INFO:teuthology.orchestra.run.smithi012.stdout:setting min_mon_release = octopus 2024-04-26T04:48:15.755 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/monmaptool: set fsid to 0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T04:48:15.755 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-26T04:48:15.755 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:48:15.755 INFO:teuthology.orchestra.run.smithi012.stdout:Creating mon... 2024-04-26T04:48:16.619 INFO:teuthology.orchestra.run.smithi012.stdout:create mon.a on 2024-04-26T04:48:16.840 INFO:teuthology.orchestra.run.smithi012.stdout:systemctl: stderr Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2024-04-26T04:48:16.989 INFO:teuthology.orchestra.run.smithi012.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-04-26T04:48:17.146 INFO:teuthology.orchestra.run.smithi012.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-0b9350c6-0388-11ef-bc93-c7b262605968.target → /etc/systemd/system/ceph-0b9350c6-0388-11ef-bc93-c7b262605968.target. 2024-04-26T04:48:17.146 INFO:teuthology.orchestra.run.smithi012.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-0b9350c6-0388-11ef-bc93-c7b262605968.target → /etc/systemd/system/ceph-0b9350c6-0388-11ef-bc93-c7b262605968.target. 2024-04-26T04:48:17.333 INFO:teuthology.orchestra.run.smithi012.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mon.a 2024-04-26T04:48:17.333 INFO:teuthology.orchestra.run.smithi012.stdout:systemctl: stderr Failed to reset failed state of unit ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mon.a.service: Unit ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mon.a.service not loaded. 2024-04-26T04:48:17.504 INFO:teuthology.orchestra.run.smithi012.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-0b9350c6-0388-11ef-bc93-c7b262605968.target.wants/ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mon.a.service → /etc/systemd/system/ceph-0b9350c6-0388-11ef-bc93-c7b262605968@.service. 2024-04-26T04:48:18.025 INFO:teuthology.orchestra.run.smithi012.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-26T04:48:18.026 INFO:teuthology.orchestra.run.smithi012.stdout:systemctl: stdout disabled 2024-04-26T04:48:18.033 INFO:teuthology.orchestra.run.smithi012.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-26T04:48:18.033 INFO:teuthology.orchestra.run.smithi012.stdout:systemctl: stdout inactive 2024-04-26T04:48:18.033 INFO:teuthology.orchestra.run.smithi012.stdout:firewalld.service is not enabled 2024-04-26T04:48:18.033 INFO:teuthology.orchestra.run.smithi012.stdout:Not possible to enable service . firewalld.service is not available 2024-04-26T04:48:18.034 INFO:teuthology.orchestra.run.smithi012.stdout:Waiting for mon to start... 2024-04-26T04:48:18.034 INFO:teuthology.orchestra.run.smithi012.stdout:Waiting for mon... 2024-04-26T04:48:19.147 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout cluster: 2024-04-26T04:48:19.147 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout id: 0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T04:48:19.147 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-04-26T04:48:19.147 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout 2024-04-26T04:48:19.147 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout services: 2024-04-26T04:48:19.147 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.592077s) 2024-04-26T04:48:19.147 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-04-26T04:48:19.147 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-04-26T04:48:19.147 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout 2024-04-26T04:48:19.147 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout data: 2024-04-26T04:48:19.148 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-04-26T04:48:19.148 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-04-26T04:48:19.148 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-04-26T04:48:19.148 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout pgs: 2024-04-26T04:48:19.148 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout 2024-04-26T04:48:19.148 INFO:teuthology.orchestra.run.smithi012.stdout:mon is available 2024-04-26T04:48:19.148 INFO:teuthology.orchestra.run.smithi012.stdout:Assimilating anything we can from ceph.conf... 2024-04-26T04:48:20.262 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout 2024-04-26T04:48:20.262 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout [global] 2024-04-26T04:48:20.262 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout fsid = 0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T04:48:20.263 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.12:3300,v1:172.21.15.12:6789] 2024-04-26T04:48:20.263 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-04-26T04:48:20.263 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-04-26T04:48:20.263 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-04-26T04:48:20.263 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-04-26T04:48:20.263 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout 2024-04-26T04:48:20.263 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout [mgr] 2024-04-26T04:48:20.263 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-04-26T04:48:20.263 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout 2024-04-26T04:48:20.263 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout [osd] 2024-04-26T04:48:20.263 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-04-26T04:48:20.263 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-04-26T04:48:20.263 INFO:teuthology.orchestra.run.smithi012.stdout:Generating new minimal ceph.conf... 2024-04-26T04:48:21.335 INFO:teuthology.orchestra.run.smithi012.stdout:Restarting the monitor... 2024-04-26T04:48:21.628 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:21 smithi012 systemd[1]: Stopping Ceph mon.a for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:48:21.628 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:21 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-a[26239]: 2024-04-26T04:48:21.625+0000 7fd2dae3e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:48:21.628 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:21 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-a[26239]: 2024-04-26T04:48:21.625+0000 7fd2dae3e700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-04-26T04:48:21.968 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:21 smithi012 podman[26489]: 2024-04-26 04:48:21.749551008 +0000 UTC m=+0.242080798 container died d5797091d0f99a0673cee8bb7b33b5a4e2c7d1b1f974b3f70b86f14bd94adba7 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-a, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=quincy-c61472c, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-26T04:48:22.253 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 podman[26489]: 2024-04-26 04:48:22.058777279 +0000 UTC m=+0.551307065 container cleanup d5797091d0f99a0673cee8bb7b33b5a4e2c7d1b1f974b3f70b86f14bd94adba7 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-a, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, RELEASE=quincy-c61472c, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD) 2024-04-26T04:48:22.253 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 bash[26489]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-a 2024-04-26T04:48:22.512 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 podman[26503]: 2024-04-26 04:48:22.252581147 +0000 UTC m=+0.502318094 container remove d5797091d0f99a0673cee8bb7b33b5a4e2c7d1b1f974b3f70b86f14bd94adba7 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-a, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-c61472c, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424) 2024-04-26T04:48:22.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mon.a.service: Deactivated successfully. 2024-04-26T04:48:22.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 systemd[1]: Stopped Ceph mon.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:48:22.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 systemd[1]: Starting Ceph mon.a for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:48:22.964 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 podman[26598]: 2024-04-26 04:48:22.686645774 +0000 UTC m=+0.098543979 container create 1b1568ad49e73ce545a5c9e562b88e4f3f39191857e81d1d96b94520b6769f62 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-a, GIT_CLEAN=True, RELEASE=quincy-c61472c, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424) 2024-04-26T04:48:22.964 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 podman[26598]: 2024-04-26 04:48:22.60631313 +0000 UTC m=+0.018211347 image pull 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 quay.ceph.io/ceph-ci/ceph:quincy 2024-04-26T04:48:22.965 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 podman[26598]: 2024-04-26 04:48:22.88053872 +0000 UTC m=+0.292436925 container init 1b1568ad49e73ce545a5c9e562b88e4f3f39191857e81d1d96b94520b6769f62 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-a, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=quincy-c61472c, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-04-26T04:48:22.965 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 podman[26598]: 2024-04-26 04:48:22.883457951 +0000 UTC m=+0.295356156 container start 1b1568ad49e73ce545a5c9e562b88e4f3f39191857e81d1d96b94520b6769f62 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-a, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-c61472c, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-04-26T04:48:22.965 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: set uid:gid to 167:167 (ceph:ceph) 2024-04-26T04:48:22.965 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable), process ceph-mon, pid 2 2024-04-26T04:48:22.965 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: pidfile_write: ignore empty --pid-file 2024-04-26T04:48:22.965 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: load: jerasure load: lrc 2024-04-26T04:48:22.965 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: RocksDB version: 6.15.5 2024-04-26T04:48:22.965 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-26T04:48:22.965 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Compile date Apr 24 2024 2024-04-26T04:48:22.965 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: DB SUMMARY 2024-04-26T04:48:22.965 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: DB Session ID: 370VV2QWXD6QD7YRO8YY 2024-04-26T04:48:22.965 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: CURRENT file: CURRENT 2024-04-26T04:48:22.965 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: IDENTITY file: IDENTITY 2024-04-26T04:48:22.966 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-04-26T04:48:22.966 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-04-26T04:48:22.966 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 84229 ; 2024-04-26T04:48:22.966 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.error_if_exists: 0 2024-04-26T04:48:22.966 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.create_if_missing: 0 2024-04-26T04:48:22.966 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.paranoid_checks: 1 2024-04-26T04:48:22.967 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-26T04:48:22.967 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.env: 0x55993a4c3b00 2024-04-26T04:48:22.967 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.fs: Posix File System 2024-04-26T04:48:22.967 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.info_log: 0x55993c986ee0 2024-04-26T04:48:22.967 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.max_file_opening_threads: 16 2024-04-26T04:48:22.967 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.statistics: (nil) 2024-04-26T04:48:22.967 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.use_fsync: 0 2024-04-26T04:48:22.967 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.max_log_file_size: 0 2024-04-26T04:48:22.967 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-26T04:48:22.967 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-26T04:48:22.967 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.keep_log_file_num: 1000 2024-04-26T04:48:22.967 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.recycle_log_file_num: 0 2024-04-26T04:48:22.968 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.allow_fallocate: 1 2024-04-26T04:48:22.968 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.allow_mmap_reads: 0 2024-04-26T04:48:22.968 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.allow_mmap_writes: 0 2024-04-26T04:48:22.968 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.use_direct_reads: 0 2024-04-26T04:48:22.968 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T04:48:22.968 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T04:48:22.968 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.db_log_dir: 2024-04-26T04:48:22.968 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-04-26T04:48:22.968 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T04:48:22.968 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T04:48:22.968 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T04:48:22.968 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T04:48:22.969 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T04:48:22.969 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T04:48:22.969 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T04:48:22.969 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T04:48:22.969 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.write_buffer_manager: 0x55993d6f6990 2024-04-26T04:48:22.969 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T04:48:22.969 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-26T04:48:22.969 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T04:48:22.969 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T04:48:22.969 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T04:48:22.969 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T04:48:22.969 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T04:48:22.969 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T04:48:22.969 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T04:48:22.970 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.unordered_write: 0 2024-04-26T04:48:22.970 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T04:48:22.970 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T04:48:22.970 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T04:48:22.970 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T04:48:22.970 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.row_cache: None 2024-04-26T04:48:22.970 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.wal_filter: None 2024-04-26T04:48:22.970 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T04:48:22.970 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T04:48:22.970 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.preserve_deletes: 0 2024-04-26T04:48:22.970 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.two_write_queues: 0 2024-04-26T04:48:22.970 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T04:48:22.970 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.atomic_flush: 0 2024-04-26T04:48:22.971 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T04:48:22.971 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T04:48:22.971 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T04:48:22.971 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.log_readahead_size: 0 2024-04-26T04:48:22.971 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T04:48:22.971 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T04:48:22.971 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T04:48:22.971 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T04:48:22.971 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T04:48:22.971 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T04:48:22.971 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.max_background_jobs: 2 2024-04-26T04:48:22.971 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.max_background_compactions: -1 2024-04-26T04:48:22.971 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.max_subcompactions: 1 2024-04-26T04:48:22.971 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T04:48:22.971 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T04:48:22.972 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T04:48:22.972 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T04:48:22.972 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T04:48:22.972 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T04:48:22.972 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T04:48:22.972 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T04:48:22.972 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.max_open_files: -1 2024-04-26T04:48:22.972 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T04:48:22.972 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T04:48:22.972 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T04:48:22.972 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T04:48:22.972 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.max_background_flushes: -1 2024-04-26T04:48:22.972 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Compression algorithms supported: 2024-04-26T04:48:22.972 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T04:48:22.973 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: kZSTD supported: 0 2024-04-26T04:48:22.973 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: kXpressCompression supported: 0 2024-04-26T04:48:22.973 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T04:48:22.973 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: kZlibCompression supported: 1 2024-04-26T04:48:22.973 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: kSnappyCompression supported: 1 2024-04-26T04:48:22.973 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: kLZ4Compression supported: 1 2024-04-26T04:48:22.973 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: kBZip2Compression supported: 0 2024-04-26T04:48:22.973 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T04:48:22.973 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-04-26T04:48:22.973 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-04-26T04:48:22.973 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T04:48:22.973 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.merge_operator: 2024-04-26T04:48:22.973 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.compaction_filter: None 2024-04-26T04:48:22.973 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T04:48:22.974 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T04:48:22.974 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T04:48:22.974 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T04:48:22.974 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55993c9d6f58) 2024-04-26T04:48:22.974 INFO:journalctl@ceph.mon.a.smithi012.stdout: cache_index_and_filter_blocks: 1 2024-04-26T04:48:22.974 INFO:journalctl@ceph.mon.a.smithi012.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T04:48:22.974 INFO:journalctl@ceph.mon.a.smithi012.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T04:48:22.974 INFO:journalctl@ceph.mon.a.smithi012.stdout: pin_top_level_index_and_filter: 1 2024-04-26T04:48:22.974 INFO:journalctl@ceph.mon.a.smithi012.stdout: index_type: 0 2024-04-26T04:48:22.974 INFO:journalctl@ceph.mon.a.smithi012.stdout: data_block_index_type: 0 2024-04-26T04:48:22.974 INFO:journalctl@ceph.mon.a.smithi012.stdout: index_shortening: 1 2024-04-26T04:48:22.974 INFO:journalctl@ceph.mon.a.smithi012.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T04:48:22.974 INFO:journalctl@ceph.mon.a.smithi012.stdout: hash_index_allow_collision: 1 2024-04-26T04:48:22.974 INFO:journalctl@ceph.mon.a.smithi012.stdout: checksum: 1 2024-04-26T04:48:22.975 INFO:journalctl@ceph.mon.a.smithi012.stdout: no_block_cache: 0 2024-04-26T04:48:22.975 INFO:journalctl@ceph.mon.a.smithi012.stdout: block_cache: 0x55993ca09090 2024-04-26T04:48:22.975 INFO:journalctl@ceph.mon.a.smithi012.stdout: block_cache_name: BinnedLRUCache 2024-04-26T04:48:22.975 INFO:journalctl@ceph.mon.a.smithi012.stdout: block_cache_options: 2024-04-26T04:48:22.975 INFO:journalctl@ceph.mon.a.smithi012.stdout: capacity : 536870912 2024-04-26T04:48:22.975 INFO:journalctl@ceph.mon.a.smithi012.stdout: num_shard_bits : 4 2024-04-26T04:48:22.975 INFO:journalctl@ceph.mon.a.smithi012.stdout: strict_capacity_limit : 0 2024-04-26T04:48:22.975 INFO:journalctl@ceph.mon.a.smithi012.stdout: high_pri_pool_ratio: 0.000 2024-04-26T04:48:22.975 INFO:journalctl@ceph.mon.a.smithi012.stdout: block_cache_compressed: (nil) 2024-04-26T04:48:22.975 INFO:journalctl@ceph.mon.a.smithi012.stdout: persistent_cache: (nil) 2024-04-26T04:48:22.975 INFO:journalctl@ceph.mon.a.smithi012.stdout: block_size: 4096 2024-04-26T04:48:22.975 INFO:journalctl@ceph.mon.a.smithi012.stdout: block_size_deviation: 10 2024-04-26T04:48:22.975 INFO:journalctl@ceph.mon.a.smithi012.stdout: block_restart_interval: 16 2024-04-26T04:48:22.975 INFO:journalctl@ceph.mon.a.smithi012.stdout: index_block_restart_interval: 1 2024-04-26T04:48:22.976 INFO:journalctl@ceph.mon.a.smithi012.stdout: metadata_block_size: 4096 2024-04-26T04:48:22.976 INFO:journalctl@ceph.mon.a.smithi012.stdout: partition_filters: 0 2024-04-26T04:48:22.976 INFO:journalctl@ceph.mon.a.smithi012.stdout: use_delta_encoding: 1 2024-04-26T04:48:22.976 INFO:journalctl@ceph.mon.a.smithi012.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-26T04:48:22.976 INFO:journalctl@ceph.mon.a.smithi012.stdout: whole_key_filtering: 1 2024-04-26T04:48:22.976 INFO:journalctl@ceph.mon.a.smithi012.stdout: verify_compression: 0 2024-04-26T04:48:22.976 INFO:journalctl@ceph.mon.a.smithi012.stdout: read_amp_bytes_per_bit: 0 2024-04-26T04:48:22.977 INFO:journalctl@ceph.mon.a.smithi012.stdout: format_version: 4 2024-04-26T04:48:22.977 INFO:journalctl@ceph.mon.a.smithi012.stdout: enable_index_compression: 1 2024-04-26T04:48:22.977 INFO:journalctl@ceph.mon.a.smithi012.stdout: block_align: 0 2024-04-26T04:48:22.977 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T04:48:22.977 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T04:48:22.977 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.compression: NoCompression 2024-04-26T04:48:22.977 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T04:48:22.977 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T04:48:22.977 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T04:48:22.977 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.num_levels: 7 2024-04-26T04:48:22.977 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T04:48:22.977 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T04:48:22.977 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T04:48:22.977 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T04:48:22.978 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T04:48:22.978 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T04:48:22.978 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T04:48:22.978 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:48:22.978 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T04:48:22.978 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T04:48:22.978 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T04:48:22.978 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T04:48:22.978 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T04:48:22.978 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T04:48:22.978 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:48:22.978 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T04:48:22.978 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T04:48:22.979 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T04:48:22.979 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T04:48:22.979 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T04:48:22.979 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T04:48:22.979 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T04:48:22.979 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T04:48:22.979 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T04:48:22.979 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T04:48:22.979 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T04:48:22.979 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T04:48:22.979 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T04:48:22.979 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T04:48:22.979 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T04:48:22.979 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T04:48:22.980 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T04:48:22.980 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T04:48:22.980 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T04:48:22.980 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.arena_block_size: 4194304 2024-04-26T04:48:22.980 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T04:48:22.980 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T04:48:22.980 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-26T04:48:22.980 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T04:48:22.980 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T04:48:22.980 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T04:48:22.980 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T04:48:22.980 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T04:48:22.980 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T04:48:22.980 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T04:48:22.980 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T04:48:22.981 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T04:48:22.981 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T04:48:22.981 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T04:48:22.981 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.table_properties_collectors: 2024-04-26T04:48:22.981 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.inplace_update_support: 0 2024-04-26T04:48:22.981 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T04:48:22.981 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T04:48:22.981 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T04:48:22.981 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T04:48:22.981 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.bloom_locality: 0 2024-04-26T04:48:22.981 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.max_successive_merges: 0 2024-04-26T04:48:22.982 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T04:48:22.982 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T04:48:22.982 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T04:48:22.982 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T04:48:22.982 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.ttl: 2592000 2024-04-26T04:48:22.982 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T04:48:22.982 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.enable_blob_files: false 2024-04-26T04:48:22.982 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.min_blob_size: 0 2024-04-26T04:48:22.982 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T04:48:22.982 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T04:48:22.982 INFO:teuthology.orchestra.run.smithi012.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2024-04-26T04:48:22.983 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T04:48:22.983 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T04:48:22.983 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T04:48:22.983 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2024-04-26T04:48:22.983 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2024-04-26T04:48:22.984 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106902909260, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-04-26T04:48:22.984 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2024-04-26T04:48:22.984 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-26T04:48:22.984 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106902910007, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80661, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78922, "index_size": 227, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10267, "raw_average_key_size": 47, "raw_value_size": 73131, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 215, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1714106902, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "1d68161f-8608-4b5c-829e-6155786e8c51", "db_session_id": "370VV2QWXD6QD7YRO8YY"}} 2024-04-26T04:48:22.984 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2024-04-26T04:48:22.984 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106902910341, "job": 1, "event": "recovery_finished"} 2024-04-26T04:48:22.984 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T04:48:22.984 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55993c99d180 2024-04-26T04:48:22.984 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: DB pointer 0x55993ca72000 2024-04-26T04:48:22.984 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-04-26T04:48:22.984 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-26T04:48:22.984 INFO:journalctl@ceph.mon.a.smithi012.stdout: ** DB Stats ** 2024-04-26T04:48:22.984 INFO:journalctl@ceph.mon.a.smithi012.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:48:22.984 INFO:journalctl@ceph.mon.a.smithi012.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T04:48:22.985 INFO:journalctl@ceph.mon.a.smithi012.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T04:48:22.985 INFO:journalctl@ceph.mon.a.smithi012.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:48:22.985 INFO:journalctl@ceph.mon.a.smithi012.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T04:48:22.985 INFO:journalctl@ceph.mon.a.smithi012.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-26T04:48:22.985 INFO:journalctl@ceph.mon.a.smithi012.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:48:22.985 INFO:journalctl@ceph.mon.a.smithi012.stdout: 2024-04-26T04:48:22.985 INFO:journalctl@ceph.mon.a.smithi012.stdout: ** Compaction Stats [default] ** 2024-04-26T04:48:22.985 INFO:journalctl@ceph.mon.a.smithi012.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:48:22.985 INFO:journalctl@ceph.mon.a.smithi012.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:48:22.985 INFO:journalctl@ceph.mon.a.smithi012.stdout: L0 2/0 80.53 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 157.6 0.00 0.00 1 0.000 0 0 2024-04-26T04:48:22.985 INFO:journalctl@ceph.mon.a.smithi012.stdout: Sum 2/0 80.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 157.6 0.00 0.00 1 0.000 0 0 2024-04-26T04:48:22.985 INFO:journalctl@ceph.mon.a.smithi012.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 157.6 0.00 0.00 1 0.000 0 0 2024-04-26T04:48:22.985 INFO:journalctl@ceph.mon.a.smithi012.stdout: 2024-04-26T04:48:22.985 INFO:journalctl@ceph.mon.a.smithi012.stdout: ** Compaction Stats [default] ** 2024-04-26T04:48:22.985 INFO:journalctl@ceph.mon.a.smithi012.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:48:22.986 INFO:journalctl@ceph.mon.a.smithi012.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:48:22.986 INFO:journalctl@ceph.mon.a.smithi012.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 157.6 0.00 0.00 1 0.000 0 0 2024-04-26T04:48:22.986 INFO:journalctl@ceph.mon.a.smithi012.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:48:22.986 INFO:journalctl@ceph.mon.a.smithi012.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-26T04:48:22.986 INFO:journalctl@ceph.mon.a.smithi012.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T04:48:22.986 INFO:journalctl@ceph.mon.a.smithi012.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T04:48:22.986 INFO:journalctl@ceph.mon.a.smithi012.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T04:48:22.986 INFO:journalctl@ceph.mon.a.smithi012.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T04:48:22.986 INFO:journalctl@ceph.mon.a.smithi012.stdout: Cumulative compaction: 0.00 GB write, 25.41 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:48:22.986 INFO:journalctl@ceph.mon.a.smithi012.stdout: Interval compaction: 0.00 GB write, 25.41 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:48:22.986 INFO:journalctl@ceph.mon.a.smithi012.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T04:48:22.986 INFO:journalctl@ceph.mon.a.smithi012.stdout: 2024-04-26T04:48:22.986 INFO:journalctl@ceph.mon.a.smithi012.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T04:48:22.986 INFO:journalctl@ceph.mon.a.smithi012.stdout: 2024-04-26T04:48:22.987 INFO:journalctl@ceph.mon.a.smithi012.stdout: ** Compaction Stats [default] ** 2024-04-26T04:48:22.987 INFO:journalctl@ceph.mon.a.smithi012.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:48:22.987 INFO:journalctl@ceph.mon.a.smithi012.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:48:22.987 INFO:journalctl@ceph.mon.a.smithi012.stdout: L0 2/0 80.53 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 157.6 0.00 0.00 1 0.000 0 0 2024-04-26T04:48:22.987 INFO:journalctl@ceph.mon.a.smithi012.stdout: Sum 2/0 80.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 157.6 0.00 0.00 1 0.000 0 0 2024-04-26T04:48:22.987 INFO:journalctl@ceph.mon.a.smithi012.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-26T04:48:22.987 INFO:journalctl@ceph.mon.a.smithi012.stdout: 2024-04-26T04:48:22.987 INFO:journalctl@ceph.mon.a.smithi012.stdout: ** Compaction Stats [default] ** 2024-04-26T04:48:22.987 INFO:journalctl@ceph.mon.a.smithi012.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:48:22.987 INFO:journalctl@ceph.mon.a.smithi012.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:48:22.987 INFO:journalctl@ceph.mon.a.smithi012.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 157.6 0.00 0.00 1 0.000 0 0 2024-04-26T04:48:22.987 INFO:journalctl@ceph.mon.a.smithi012.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:48:22.987 INFO:journalctl@ceph.mon.a.smithi012.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-26T04:48:22.987 INFO:journalctl@ceph.mon.a.smithi012.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T04:48:22.988 INFO:journalctl@ceph.mon.a.smithi012.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T04:48:22.988 INFO:journalctl@ceph.mon.a.smithi012.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T04:48:22.988 INFO:journalctl@ceph.mon.a.smithi012.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T04:48:22.988 INFO:journalctl@ceph.mon.a.smithi012.stdout: Cumulative compaction: 0.00 GB write, 25.14 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:48:22.988 INFO:journalctl@ceph.mon.a.smithi012.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:48:22.988 INFO:journalctl@ceph.mon.a.smithi012.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T04:48:22.988 INFO:journalctl@ceph.mon.a.smithi012.stdout: 2024-04-26T04:48:22.988 INFO:journalctl@ceph.mon.a.smithi012.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T04:48:22.988 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: starting mon.a rank 0 at public addrs [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] at bind addrs [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T04:48:22.988 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: mon.a@-1(???) e1 preinit fsid 0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T04:48:22.988 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: mon.a@-1(???).mds e1 new map 2024-04-26T04:48:22.988 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: mon.a@-1(???).mds e1 print_map 2024-04-26T04:48:22.988 INFO:journalctl@ceph.mon.a.smithi012.stdout: e1 2024-04-26T04:48:22.988 INFO:journalctl@ceph.mon.a.smithi012.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T04:48:22.988 INFO:journalctl@ceph.mon.a.smithi012.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T04:48:22.989 INFO:journalctl@ceph.mon.a.smithi012.stdout: legacy client fscid: -1 2024-04-26T04:48:22.989 INFO:journalctl@ceph.mon.a.smithi012.stdout: 2024-04-26T04:48:22.989 INFO:journalctl@ceph.mon.a.smithi012.stdout: No filesystems configured 2024-04-26T04:48:22.989 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-26T04:48:22.989 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:48:22.989 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:48:22.989 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:48:22.989 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-04-26T04:48:22.989 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 bash[26598]: 1b1568ad49e73ce545a5c9e562b88e4f3f39191857e81d1d96b94520b6769f62 2024-04-26T04:48:22.989 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 systemd[1]: Started Ceph mon.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:48:23.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-26T04:48:23.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: monmap e1: 1 mons at {a=[v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:48:23.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: fsmap 2024-04-26T04:48:23.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: osdmap e1: 0 total, 0 up, 0 in 2024-04-26T04:48:23.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:22 smithi012 ceph-mon[26617]: mgrmap e1: no daemons active 2024-04-26T04:48:23.876 INFO:teuthology.orchestra.run.smithi012.stdout:Wrote config to /etc/ceph/ceph.conf 2024-04-26T04:48:23.882 INFO:teuthology.orchestra.run.smithi012.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-04-26T04:48:23.882 INFO:teuthology.orchestra.run.smithi012.stdout:Creating mgr... 2024-04-26T04:48:23.883 INFO:teuthology.orchestra.run.smithi012.stdout:Verifying port 9283 ... 2024-04-26T04:48:24.062 INFO:teuthology.orchestra.run.smithi012.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mgr.y 2024-04-26T04:48:24.062 INFO:teuthology.orchestra.run.smithi012.stdout:systemctl: stderr Failed to reset failed state of unit ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mgr.y.service: Unit ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mgr.y.service not loaded. 2024-04-26T04:48:24.222 INFO:teuthology.orchestra.run.smithi012.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-0b9350c6-0388-11ef-bc93-c7b262605968.target.wants/ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-0b9350c6-0388-11ef-bc93-c7b262605968@.service. 2024-04-26T04:48:24.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:24 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1345658623' entity='client.admin' 2024-04-26T04:48:24.783 INFO:teuthology.orchestra.run.smithi012.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-26T04:48:24.783 INFO:teuthology.orchestra.run.smithi012.stdout:systemctl: stdout disabled 2024-04-26T04:48:24.791 INFO:teuthology.orchestra.run.smithi012.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-26T04:48:24.791 INFO:teuthology.orchestra.run.smithi012.stdout:systemctl: stdout inactive 2024-04-26T04:48:24.791 INFO:teuthology.orchestra.run.smithi012.stdout:firewalld.service is not enabled 2024-04-26T04:48:24.791 INFO:teuthology.orchestra.run.smithi012.stdout:Not possible to enable service . firewalld.service is not available 2024-04-26T04:48:24.800 INFO:teuthology.orchestra.run.smithi012.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-26T04:48:24.800 INFO:teuthology.orchestra.run.smithi012.stdout:systemctl: stdout disabled 2024-04-26T04:48:24.808 INFO:teuthology.orchestra.run.smithi012.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-26T04:48:24.808 INFO:teuthology.orchestra.run.smithi012.stdout:systemctl: stdout inactive 2024-04-26T04:48:24.808 INFO:teuthology.orchestra.run.smithi012.stdout:firewalld.service is not enabled 2024-04-26T04:48:24.808 INFO:teuthology.orchestra.run.smithi012.stdout:Not possible to open ports <[9283]>. firewalld.service is not available 2024-04-26T04:48:24.808 INFO:teuthology.orchestra.run.smithi012.stdout:Waiting for mgr to start... 2024-04-26T04:48:24.808 INFO:teuthology.orchestra.run.smithi012.stdout:Waiting for mgr... 2024-04-26T04:48:24.928 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:24 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:24.819+0000 7f58dd5ea1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:48:25.196 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:24 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:24.990+0000 7f58dd5ea1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:48:25.196 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:25 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:25.097+0000 7f58dd5ea1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:48:25.468 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:25 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:25.423+0000 7f58dd5ea1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:48:25.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:25 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/2674394251' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-26T04:48:25.763 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:25 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:25.508+0000 7f58dd5ea1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:48:25.897 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout 2024-04-26T04:48:25.897 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout { 2024-04-26T04:48:25.897 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "fsid": "0b9350c6-0388-11ef-bc93-c7b262605968", 2024-04-26T04:48:25.897 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "health": { 2024-04-26T04:48:25.898 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-26T04:48:25.898 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-26T04:48:25.898 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-26T04:48:25.898 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:48:25.898 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-26T04:48:25.898 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-26T04:48:25.898 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout 0 2024-04-26T04:48:25.898 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:48:25.898 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-26T04:48:25.898 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "a" 2024-04-26T04:48:25.898 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:48:25.899 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2024-04-26T04:48:25.899 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-26T04:48:25.899 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:48:25.899 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-04-26T04:48:25.899 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-26T04:48:25.899 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:48:25.899 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-26T04:48:25.899 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:48:25.899 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-26T04:48:25.899 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-26T04:48:25.899 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-26T04:48:25.899 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-26T04:48:25.900 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-26T04:48:25.900 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-26T04:48:25.901 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:48:25.901 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-26T04:48:25.901 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-26T04:48:25.901 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-26T04:48:25.901 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-26T04:48:25.901 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-26T04:48:25.901 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-26T04:48:25.901 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-26T04:48:25.901 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-26T04:48:25.901 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-26T04:48:25.901 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:48:25.902 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-26T04:48:25.902 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:48:25.902 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-26T04:48:25.902 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-26T04:48:25.902 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:48:25.902 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-26T04:48:25.902 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "available": false, 2024-04-26T04:48:25.902 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-26T04:48:25.902 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-26T04:48:25.903 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-26T04:48:25.903 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-26T04:48:25.903 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "restful" 2024-04-26T04:48:25.903 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:48:25.903 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-26T04:48:25.903 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:48:25.903 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-26T04:48:25.903 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:48:25.903 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "modified": "2024-04-26T04:48:17.967233+0000", 2024-04-26T04:48:25.903 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-26T04:48:25.903 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:48:25.903 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-26T04:48:25.903 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout } 2024-04-26T04:48:25.903 INFO:teuthology.orchestra.run.smithi012.stdout:mgr not available, waiting (1/15)... 2024-04-26T04:48:27.012 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:26 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:26.736+0000 7f58dd5ea1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:48:27.013 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:26 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:26.904+0000 7f58dd5ea1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:48:27.345 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:27 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:27.077+0000 7f58dd5ea1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:48:27.346 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:27 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:27.344+0000 7f58dd5ea1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:48:27.762 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:27 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:27.436+0000 7f58dd5ea1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:48:27.763 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:27 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:27.532+0000 7f58dd5ea1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:48:28.153 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:27 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:27.815+0000 7f58dd5ea1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:48:28.413 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:28 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:28.313+0000 7f58dd5ea1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:48:28.705 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:28 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/4066203173' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-26T04:48:28.957 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:28 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:28.864+0000 7f58dd5ea1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:48:28.960 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout 2024-04-26T04:48:28.960 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout { 2024-04-26T04:48:28.961 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "fsid": "0b9350c6-0388-11ef-bc93-c7b262605968", 2024-04-26T04:48:28.961 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "health": { 2024-04-26T04:48:28.961 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-26T04:48:28.961 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-26T04:48:28.961 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-26T04:48:28.961 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:48:28.961 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-26T04:48:28.961 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-26T04:48:28.961 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout 0 2024-04-26T04:48:28.961 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:48:28.961 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-26T04:48:28.961 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "a" 2024-04-26T04:48:28.961 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:48:28.962 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "quorum_age": 5, 2024-04-26T04:48:28.962 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-26T04:48:28.962 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:48:28.962 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-04-26T04:48:28.962 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-26T04:48:28.962 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:48:28.962 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-26T04:48:28.962 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:48:28.962 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-26T04:48:28.962 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-26T04:48:28.962 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-26T04:48:28.962 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-26T04:48:28.962 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-26T04:48:28.962 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-26T04:48:28.966 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:48:28.966 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-26T04:48:28.966 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-26T04:48:28.966 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-26T04:48:28.966 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-26T04:48:28.966 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-26T04:48:28.966 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-26T04:48:28.966 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-26T04:48:28.966 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-26T04:48:28.966 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-26T04:48:28.966 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:48:28.966 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-26T04:48:28.966 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:48:28.967 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-26T04:48:28.967 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-26T04:48:28.967 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:48:28.967 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-26T04:48:28.967 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "available": false, 2024-04-26T04:48:28.967 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-26T04:48:28.967 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-26T04:48:28.967 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-26T04:48:28.967 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-26T04:48:28.967 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "restful" 2024-04-26T04:48:28.967 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:48:28.967 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-26T04:48:28.967 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:48:28.967 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-26T04:48:28.967 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:48:28.968 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "modified": "2024-04-26T04:48:17.967233+0000", 2024-04-26T04:48:28.968 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-26T04:48:28.968 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:48:28.968 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-26T04:48:28.968 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout } 2024-04-26T04:48:28.968 INFO:teuthology.orchestra.run.smithi012.stdout:mgr not available, waiting (2/15)... 2024-04-26T04:48:29.262 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:28 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:28.961+0000 7f58dd5ea1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:48:29.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:29 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:29.143+0000 7f58dd5ea1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:48:29.709 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:29 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:29.444+0000 7f58dd5ea1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:48:29.709 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:29 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:29.707+0000 7f58dd5ea1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:48:30.012 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:29 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:29.952+0000 7f58dd5ea1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:48:30.392 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:30 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:30.087+0000 7f58dd5ea1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:48:30.392 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:30 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:30.182+0000 7f58dd5ea1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:48:30.392 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:30 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:30.300+0000 7f58dd5ea1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:48:30.392 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:30 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:30.390+0000 7f58dd5ea1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:48:31.341 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:31 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:31.222+0000 7f58dd5ea1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:48:31.762 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:31 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:31.421+0000 7f58dd5ea1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:48:31.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:31 smithi012 ceph-mon[26617]: Activating manager daemon y 2024-04-26T04:48:31.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:31 smithi012 ceph-mon[26617]: mgrmap e2: y(active, starting, since 0.00325695s) 2024-04-26T04:48:31.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:31 smithi012 ceph-mon[26617]: from='mgr.14100 172.21.15.12:0/2370312467' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:48:31.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:31 smithi012 ceph-mon[26617]: from='mgr.14100 172.21.15.12:0/2370312467' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:48:31.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:31 smithi012 ceph-mon[26617]: from='mgr.14100 172.21.15.12:0/2370312467' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:48:31.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:31 smithi012 ceph-mon[26617]: from='mgr.14100 172.21.15.12:0/2370312467' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:48:31.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:31 smithi012 ceph-mon[26617]: from='mgr.14100 172.21.15.12:0/2370312467' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:48:31.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:31 smithi012 ceph-mon[26617]: Manager daemon y is now available 2024-04-26T04:48:31.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:31 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1303063711' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-26T04:48:31.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:31 smithi012 ceph-mon[26617]: from='mgr.14100 172.21.15.12:0/2370312467' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:48:31.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:31 smithi012 ceph-mon[26617]: from='mgr.14100 172.21.15.12:0/2370312467' entity='mgr.y' 2024-04-26T04:48:31.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:31 smithi012 ceph-mon[26617]: from='mgr.14100 172.21.15.12:0/2370312467' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:48:31.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:31 smithi012 ceph-mon[26617]: from='mgr.14100 172.21.15.12:0/2370312467' entity='mgr.y' 2024-04-26T04:48:31.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:31 smithi012 ceph-mon[26617]: from='mgr.14100 172.21.15.12:0/2370312467' entity='mgr.y' 2024-04-26T04:48:31.949 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout 2024-04-26T04:48:31.950 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout { 2024-04-26T04:48:31.950 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "fsid": "0b9350c6-0388-11ef-bc93-c7b262605968", 2024-04-26T04:48:31.950 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "health": { 2024-04-26T04:48:31.950 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-26T04:48:31.950 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-26T04:48:31.950 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-26T04:48:31.950 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:48:31.950 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-26T04:48:31.950 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-26T04:48:31.950 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout 0 2024-04-26T04:48:31.950 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:48:31.950 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-26T04:48:31.951 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "a" 2024-04-26T04:48:31.951 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:48:31.951 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "quorum_age": 8, 2024-04-26T04:48:31.951 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-26T04:48:31.951 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:48:31.951 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-04-26T04:48:31.951 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-26T04:48:31.951 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:48:31.951 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-26T04:48:31.951 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:48:31.951 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-26T04:48:31.951 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-26T04:48:31.951 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-26T04:48:31.951 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-26T04:48:31.951 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-26T04:48:31.952 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-26T04:48:31.955 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:48:31.955 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-26T04:48:31.955 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-26T04:48:31.955 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-26T04:48:31.955 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-26T04:48:31.955 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-26T04:48:31.955 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-26T04:48:31.955 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-26T04:48:31.955 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-26T04:48:31.955 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-26T04:48:31.955 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:48:31.956 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-26T04:48:31.956 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:48:31.956 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-26T04:48:31.956 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-26T04:48:31.956 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:48:31.956 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-26T04:48:31.956 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "available": false, 2024-04-26T04:48:31.956 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-26T04:48:31.956 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-26T04:48:31.956 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-26T04:48:31.956 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-26T04:48:31.956 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "restful" 2024-04-26T04:48:31.956 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:48:31.956 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-26T04:48:31.957 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:48:31.957 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-26T04:48:31.957 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:48:31.957 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "modified": "2024-04-26T04:48:17.967233+0000", 2024-04-26T04:48:31.957 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-26T04:48:31.957 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:48:31.957 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-26T04:48:31.957 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout } 2024-04-26T04:48:31.957 INFO:teuthology.orchestra.run.smithi012.stdout:mgr not available, waiting (3/15)... 2024-04-26T04:48:33.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:33 smithi012 ceph-mon[26617]: mgrmap e3: y(active, since 1.00779s) 2024-04-26T04:48:34.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:34 smithi012 ceph-mon[26617]: mgrmap e4: y(active, since 2s) 2024-04-26T04:48:37.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:37 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1696785698' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-26T04:48:43.276 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout 2024-04-26T04:48:43.276 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout { 2024-04-26T04:48:43.276 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "fsid": "0b9350c6-0388-11ef-bc93-c7b262605968", 2024-04-26T04:48:43.276 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "health": { 2024-04-26T04:48:43.277 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-26T04:48:43.277 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-26T04:48:43.277 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-26T04:48:43.277 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:48:43.277 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-26T04:48:43.277 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-26T04:48:43.277 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout 0 2024-04-26T04:48:43.277 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:48:43.277 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-26T04:48:43.277 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "a" 2024-04-26T04:48:43.277 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:48:43.277 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "quorum_age": 14, 2024-04-26T04:48:43.277 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-26T04:48:43.277 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:48:43.277 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-04-26T04:48:43.278 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-26T04:48:43.278 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:48:43.278 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-26T04:48:43.278 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:48:43.278 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-26T04:48:43.278 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-26T04:48:43.278 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-26T04:48:43.278 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-26T04:48:43.278 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-26T04:48:43.278 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-26T04:48:43.282 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:48:43.282 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-26T04:48:43.282 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-26T04:48:43.282 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-26T04:48:43.282 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-26T04:48:43.282 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-26T04:48:43.283 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-26T04:48:43.283 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-26T04:48:43.283 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-26T04:48:43.283 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-26T04:48:43.283 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:48:43.283 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-26T04:48:43.283 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:48:43.283 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-26T04:48:43.283 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-26T04:48:43.283 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:48:43.283 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-26T04:48:43.283 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-26T04:48:43.283 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-26T04:48:43.283 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-26T04:48:43.283 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-26T04:48:43.284 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-26T04:48:43.284 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "restful" 2024-04-26T04:48:43.284 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:48:43.284 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-26T04:48:43.284 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:48:43.284 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-26T04:48:43.284 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:48:43.284 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "modified": "2024-04-26T04:48:17.967233+0000", 2024-04-26T04:48:43.284 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-26T04:48:43.284 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:48:43.284 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-26T04:48:43.284 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout } 2024-04-26T04:48:43.284 INFO:teuthology.orchestra.run.smithi012.stdout:mgr is available 2024-04-26T04:48:44.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:44 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/2788580949' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-26T04:48:45.027 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout 2024-04-26T04:48:45.029 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout [global] 2024-04-26T04:48:45.029 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout fsid = 0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T04:48:45.029 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-04-26T04:48:45.029 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-04-26T04:48:45.029 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-04-26T04:48:45.029 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-04-26T04:48:45.029 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout 2024-04-26T04:48:45.029 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout [mgr] 2024-04-26T04:48:45.029 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-04-26T04:48:45.029 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout 2024-04-26T04:48:45.030 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout [osd] 2024-04-26T04:48:45.030 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-04-26T04:48:45.030 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-04-26T04:48:45.030 INFO:teuthology.orchestra.run.smithi012.stdout:Enabling cephadm module... 2024-04-26T04:48:46.103 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:45 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/2503155562' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-26T04:48:46.103 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:45 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: ignoring --setuser ceph since I am not root 2024-04-26T04:48:46.103 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:45 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: ignoring --setgroup ceph since I am not root 2024-04-26T04:48:46.104 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:45 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:45.936+0000 7fd9ab9c81c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:48:46.104 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:46 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:46.102+0000 7fd9ab9c81c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:48:46.397 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:46 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:46.206+0000 7fd9ab9c81c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:48:46.649 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:46 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:46.530+0000 7fd9ab9c81c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:48:46.649 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:46 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:46.615+0000 7fd9ab9c81c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:48:47.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:46 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/2503155562' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-26T04:48:47.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:46 smithi012 ceph-mon[26617]: mgrmap e5: y(active, since 14s) 2024-04-26T04:48:47.715 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout { 2024-04-26T04:48:47.716 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-04-26T04:48:47.716 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-26T04:48:47.716 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-04-26T04:48:47.716 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-04-26T04:48:47.716 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout } 2024-04-26T04:48:47.716 INFO:teuthology.orchestra.run.smithi012.stdout:Waiting for the mgr to restart... 2024-04-26T04:48:47.716 INFO:teuthology.orchestra.run.smithi012.stdout:Waiting for mgr epoch 5... 2024-04-26T04:48:48.003 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:47 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1935999059' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-26T04:48:48.003 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:47 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:47.828+0000 7fd9ab9c81c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:48:48.262 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:48 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:48.001+0000 7fd9ab9c81c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:48:48.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:48 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:48.176+0000 7fd9ab9c81c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:48:48.763 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:48 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:48.445+0000 7fd9ab9c81c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:48:48.763 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:48 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:48.537+0000 7fd9ab9c81c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:48:48.763 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:48 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:48.633+0000 7fd9ab9c81c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:48:49.262 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:48 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:48.915+0000 7fd9ab9c81c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:48:49.762 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:49 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:49.390+0000 7fd9ab9c81c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:48:50.205 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:49 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:49.928+0000 7fd9ab9c81c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:48:50.205 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:50 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:50.024+0000 7fd9ab9c81c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:48:50.205 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:50 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:50.204+0000 7fd9ab9c81c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:48:50.762 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:50 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:50.503+0000 7fd9ab9c81c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:48:51.147 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:50 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:50.767+0000 7fd9ab9c81c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:48:51.147 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:51 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:51.010+0000 7fd9ab9c81c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:48:51.147 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:51 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:51.146+0000 7fd9ab9c81c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:48:51.445 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:51 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:51.240+0000 7fd9ab9c81c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:48:51.445 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:51 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:51.357+0000 7fd9ab9c81c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:48:51.762 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:51 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:51.451+0000 7fd9ab9c81c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:48:52.472 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:52 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:52.276+0000 7fd9ab9c81c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:48:52.762 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:52 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:48:52.471+0000 7fd9ab9c81c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:48:52.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:52 smithi012 ceph-mon[26617]: Active manager daemon y restarted 2024-04-26T04:48:52.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:52 smithi012 ceph-mon[26617]: Activating manager daemon y 2024-04-26T04:48:52.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:52 smithi012 ceph-mon[26617]: osdmap e2: 0 total, 0 up, 0 in 2024-04-26T04:48:52.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:52 smithi012 ceph-mon[26617]: mgrmap e6: y(active, starting, since 0.00308157s) 2024-04-26T04:48:52.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:52 smithi012 ceph-mon[26617]: from='mgr.14120 172.21.15.12:0/2632730383' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:48:52.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:52 smithi012 ceph-mon[26617]: from='mgr.14120 172.21.15.12:0/2632730383' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:48:52.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:52 smithi012 ceph-mon[26617]: from='mgr.14120 172.21.15.12:0/2632730383' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:48:52.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:52 smithi012 ceph-mon[26617]: from='mgr.14120 172.21.15.12:0/2632730383' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:48:52.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:52 smithi012 ceph-mon[26617]: from='mgr.14120 172.21.15.12:0/2632730383' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:48:52.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:52 smithi012 ceph-mon[26617]: Manager daemon y is now available 2024-04-26T04:48:52.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:52 smithi012 ceph-mon[26617]: from='mgr.14120 172.21.15.12:0/2632730383' entity='mgr.y' 2024-04-26T04:48:52.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:52 smithi012 ceph-mon[26617]: from='mgr.14120 172.21.15.12:0/2632730383' entity='mgr.y' 2024-04-26T04:48:52.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:52 smithi012 ceph-mon[26617]: from='mgr.14120 172.21.15.12:0/2632730383' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:48:52.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:52 smithi012 ceph-mon[26617]: from='mgr.14120 172.21.15.12:0/2632730383' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:48:52.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:52 smithi012 ceph-mon[26617]: from='mgr.14120 172.21.15.12:0/2632730383' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:48:52.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:52 smithi012 ceph-mon[26617]: from='mgr.14120 172.21.15.12:0/2632730383' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:48:52.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:52 smithi012 ceph-mon[26617]: from='mgr.14120 172.21.15.12:0/2632730383' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:48:53.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:53 smithi012 ceph-mon[26617]: Found migration_current of "None". Setting to last migration. 2024-04-26T04:48:53.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:53 smithi012 ceph-mon[26617]: mgrmap e7: y(active, since 1.00557s) 2024-04-26T04:48:54.004 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout { 2024-04-26T04:48:54.004 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-04-26T04:48:54.004 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "initialized": true 2024-04-26T04:48:54.004 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout } 2024-04-26T04:48:54.004 INFO:teuthology.orchestra.run.smithi012.stdout:mgr epoch 5 is available 2024-04-26T04:48:54.004 INFO:teuthology.orchestra.run.smithi012.stdout:Setting orchestrator backend to cephadm... 2024-04-26T04:48:54.915 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:54 smithi012 ceph-mon[26617]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-26T04:48:54.915 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:54 smithi012 ceph-mon[26617]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-26T04:48:54.915 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:54 smithi012 ceph-mon[26617]: [26/Apr/2024:04:48:53] ENGINE Bus STARTING 2024-04-26T04:48:54.915 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:54 smithi012 ceph-mon[26617]: [26/Apr/2024:04:48:53] ENGINE Serving on https://172.21.15.12:7150 2024-04-26T04:48:54.915 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:54 smithi012 ceph-mon[26617]: [26/Apr/2024:04:48:53] ENGINE Bus STARTED 2024-04-26T04:48:54.915 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:54 smithi012 ceph-mon[26617]: from='mgr.14120 172.21.15.12:0/2632730383' entity='mgr.y' 2024-04-26T04:48:54.915 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:54 smithi012 ceph-mon[26617]: from='mgr.14120 172.21.15.12:0/2632730383' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:48:54.915 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:54 smithi012 ceph-mon[26617]: from='mgr.14120 172.21.15.12:0/2632730383' entity='mgr.y' 2024-04-26T04:48:55.765 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:55 smithi012 ceph-mon[26617]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:48:55.765 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:55 smithi012 ceph-mon[26617]: mgrmap e8: y(active, since 2s) 2024-04-26T04:48:55.766 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:55 smithi012 ceph-mon[26617]: from='mgr.14120 172.21.15.12:0/2632730383' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:48:56.375 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout value unchanged 2024-04-26T04:48:56.375 INFO:teuthology.orchestra.run.smithi012.stdout:Generating ssh key... 2024-04-26T04:48:57.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:56 smithi012 ceph-mon[26617]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:48:57.761 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:57 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: Generating public/private rsa key pair. 2024-04-26T04:48:57.761 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:57 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: Your identification has been saved in /tmp/tmph5e4vp6c/key. 2024-04-26T04:48:57.761 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:57 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: Your public key has been saved in /tmp/tmph5e4vp6c/key.pub. 2024-04-26T04:48:57.761 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:57 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: The key fingerprint is: 2024-04-26T04:48:57.761 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:57 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: SHA256:19lCa4CSB5J19vEG0nIF6f8texSI4lMHs6gunuu80Zs ceph-0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T04:48:57.761 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:57 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: The key's randomart image is: 2024-04-26T04:48:57.761 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:57 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: +---[RSA 3072]----+ 2024-04-26T04:48:57.761 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:57 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: | .oo +.++. | 2024-04-26T04:48:57.761 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:57 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: | .. =.++* | 2024-04-26T04:48:57.762 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:57 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: | o o++ O . | 2024-04-26T04:48:57.762 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:57 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: | o o.O * . | 2024-04-26T04:48:57.762 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:57 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: | S +.B . .| 2024-04-26T04:48:57.762 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:57 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: | .. + ... .| 2024-04-26T04:48:57.762 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:57 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: | ... . . o | 2024-04-26T04:48:57.762 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:57 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: | ..o.o o o| 2024-04-26T04:48:57.762 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:57 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: | oB+E .+ | 2024-04-26T04:48:57.762 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:48:57 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: +----[SHA256]-----+ 2024-04-26T04:48:58.645 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:58 smithi012 ceph-mon[26617]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:48:58.645 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:58 smithi012 ceph-mon[26617]: Generating ssh key... 2024-04-26T04:48:58.645 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:58 smithi012 ceph-mon[26617]: from='mgr.14120 172.21.15.12:0/2632730383' entity='mgr.y' 2024-04-26T04:48:58.645 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:58 smithi012 ceph-mon[26617]: from='mgr.14120 172.21.15.12:0/2632730383' entity='mgr.y' 2024-04-26T04:48:59.195 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDThcoSHXia9Nrj2bPpewcabZozpxqaZ0lAcyjLAtHIf7s8kYQUvJcJq/tzhCYL4+gkXG87f8071cQpmKspSmXfYPH4wAkx0IUVhwlTM+UQ8G3SdzcueW3zx+w3uAmgmSgVXK5nbAB8zyUXkEy7uGfigLJtkKfPjUREse5cjfpEdQlz9GvBCerT9Wbdtc9hvSdTtpkVr2vzlYNwsGt9ZF4b5XkSYtxX4g8ZrP84TJIoHo2VAF09llX/CDM+Xmc8zyfixqm+eHuhf7WYqyQZtg7xF5tMjrAmteQo2Wvbo68umDyZtZlr0Y2eXPcaUMqn7yvDN8pup+5feK4B2StVQ0WySKk7HavzBain1ihbDDYvSVQVhClocjt2nyTt5Oqss5/vPzo0hXf9oANPgjqsTW/OnSZHzQkZUDlZC3Q+NvAaaTd9IbwNdZ+RvcxbPHKFOERHNnpOaTXnS59k3cEBPEZz1ZDIC8gz2Mu0fruTO1JD+LzgSr2eVzkIt4INZdjCTu0= ceph-0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T04:48:59.196 INFO:teuthology.orchestra.run.smithi012.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-04-26T04:48:59.196 INFO:teuthology.orchestra.run.smithi012.stdout:Adding key to root@localhost authorized_keys... 2024-04-26T04:48:59.196 INFO:teuthology.orchestra.run.smithi012.stdout:Adding host smithi012... 2024-04-26T04:48:59.512 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:48:59 smithi012 ceph-mon[26617]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:49:01.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:00 smithi012 ceph-mon[26617]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi012", "addr": "172.21.15.12", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:49:01.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:00 smithi012 ceph-mon[26617]: Deploying cephadm binary to smithi012 2024-04-26T04:49:02.294 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout Added host 'smithi012' with addr '172.21.15.12' 2024-04-26T04:49:02.295 INFO:teuthology.orchestra.run.smithi012.stdout:Deploying unmanaged mon service... 2024-04-26T04:49:02.965 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:02 smithi012 ceph-mon[26617]: from='mgr.14120 172.21.15.12:0/2632730383' entity='mgr.y' 2024-04-26T04:49:02.966 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:02 smithi012 ceph-mon[26617]: Added host smithi012 2024-04-26T04:49:02.966 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:02 smithi012 ceph-mon[26617]: from='mgr.14120 172.21.15.12:0/2632730383' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:03.704 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-04-26T04:49:03.704 INFO:teuthology.orchestra.run.smithi012.stdout:Deploying unmanaged mgr service... 2024-04-26T04:49:04.230 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:03 smithi012 ceph-mon[26617]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:49:04.230 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:03 smithi012 ceph-mon[26617]: Saving service mon spec with placement count:5 2024-04-26T04:49:04.230 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:03 smithi012 ceph-mon[26617]: from='mgr.14120 172.21.15.12:0/2632730383' entity='mgr.y' 2024-04-26T04:49:05.598 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-04-26T04:49:06.247 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:05 smithi012 ceph-mon[26617]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:49:06.248 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:05 smithi012 ceph-mon[26617]: Saving service mgr spec with placement count:2 2024-04-26T04:49:06.248 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:05 smithi012 ceph-mon[26617]: from='mgr.14120 172.21.15.12:0/2632730383' entity='mgr.y' 2024-04-26T04:49:06.248 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:05 smithi012 ceph-mon[26617]: from='mgr.14120 172.21.15.12:0/2632730383' entity='mgr.y' 2024-04-26T04:49:06.248 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:05 smithi012 ceph-mon[26617]: from='mgr.14120 172.21.15.12:0/2632730383' entity='mgr.y' 2024-04-26T04:49:07.467 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:07 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/3560805716' entity='client.admin' 2024-04-26T04:49:08.230 INFO:teuthology.orchestra.run.smithi012.stdout:Enabling the dashboard module... 2024-04-26T04:49:08.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:08 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/4092178760' entity='client.admin' 2024-04-26T04:49:10.108 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:09 smithi012 ceph-mon[26617]: from='mgr.14120 172.21.15.12:0/2632730383' entity='mgr.y' 2024-04-26T04:49:10.108 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:09 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1047725457' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-26T04:49:10.109 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:09 smithi012 ceph-mon[26617]: from='mgr.14120 172.21.15.12:0/2632730383' entity='mgr.y' 2024-04-26T04:49:10.109 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:49:09 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: ignoring --setuser ceph since I am not root 2024-04-26T04:49:10.109 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:49:09 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: ignoring --setgroup ceph since I am not root 2024-04-26T04:49:10.109 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:49:09 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:49:09.938+0000 7fbf456b91c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:49:10.109 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:49:10 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:49:10.107+0000 7fbf456b91c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:49:10.512 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:49:10 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:49:10.211+0000 7fbf456b91c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:49:11.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:10 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1047725457' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-26T04:49:11.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:10 smithi012 ceph-mon[26617]: mgrmap e9: y(active, since 17s) 2024-04-26T04:49:11.013 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:49:10 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:49:10.530+0000 7fbf456b91c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:49:11.013 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:49:10 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:49:10.617+0000 7fbf456b91c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:49:12.205 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:12 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/3508967027' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-26T04:49:12.206 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:49:11 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:49:11.847+0000 7fbf456b91c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:49:12.206 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:49:12 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:49:12.025+0000 7fbf456b91c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:49:12.206 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:49:12 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:49:12.204+0000 7fbf456b91c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:49:12.592 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout { 2024-04-26T04:49:12.592 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-04-26T04:49:12.593 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-26T04:49:12.593 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-04-26T04:49:12.593 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-04-26T04:49:12.593 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout } 2024-04-26T04:49:12.593 INFO:teuthology.orchestra.run.smithi012.stdout:Waiting for the mgr to restart... 2024-04-26T04:49:12.593 INFO:teuthology.orchestra.run.smithi012.stdout:Waiting for mgr epoch 9... 2024-04-26T04:49:12.762 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:49:12 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:49:12.480+0000 7fbf456b91c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:49:12.763 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:49:12 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:49:12.575+0000 7fbf456b91c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:49:12.763 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:49:12 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:49:12.686+0000 7fbf456b91c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:49:13.262 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:49:12 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:49:12.978+0000 7fbf456b91c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:49:13.762 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:49:13 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:49:13.458+0000 7fbf456b91c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:49:14.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:49:14 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:49:13.999+0000 7fbf456b91c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:49:14.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:49:14 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:49:14.096+0000 7fbf456b91c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:49:14.582 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:49:14 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:49:14.279+0000 7fbf456b91c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:49:14.582 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:49:14 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:49:14.580+0000 7fbf456b91c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:49:15.230 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:49:14 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:49:14.844+0000 7fbf456b91c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:49:15.231 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:49:15 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:49:15.092+0000 7fbf456b91c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:49:15.231 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:49:15 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:49:15.228+0000 7fbf456b91c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:49:15.512 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:49:15 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:49:15.324+0000 7fbf456b91c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:49:15.513 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:49:15 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:49:15.441+0000 7fbf456b91c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:49:16.012 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:49:15 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:49:15.533+0000 7fbf456b91c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:49:16.612 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:49:16 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:49:16.360+0000 7fbf456b91c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:49:16.612 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:49:16 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:49:16.557+0000 7fbf456b91c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:49:16.612 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:16 smithi012 ceph-mon[26617]: Active manager daemon y restarted 2024-04-26T04:49:16.613 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:16 smithi012 ceph-mon[26617]: Activating manager daemon y 2024-04-26T04:49:16.613 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:16 smithi012 ceph-mon[26617]: osdmap e3: 0 total, 0 up, 0 in 2024-04-26T04:49:16.613 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:16 smithi012 ceph-mon[26617]: mgrmap e10: y(active, starting, since 0.00562022s) 2024-04-26T04:49:16.613 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:16 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:49:16.613 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:16 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:49:16.613 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:16 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:49:16.613 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:16 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:49:16.613 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:16 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:49:16.613 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:16 smithi012 ceph-mon[26617]: Manager daemon y is now available 2024-04-26T04:49:16.613 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:16 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:16.613 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:16 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:16.613 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:16 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:49:16.613 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:16 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:49:18.269 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout { 2024-04-26T04:49:18.269 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-04-26T04:49:18.269 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout "initialized": true 2024-04-26T04:49:18.269 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout } 2024-04-26T04:49:18.269 INFO:teuthology.orchestra.run.smithi012.stdout:mgr epoch 9 is available 2024-04-26T04:49:18.269 INFO:teuthology.orchestra.run.smithi012.stdout:Generating a dashboard self-signed certificate... 2024-04-26T04:49:18.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:18 smithi012 ceph-mon[26617]: [26/Apr/2024:04:49:17] ENGINE Bus STARTING 2024-04-26T04:49:18.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:18 smithi012 ceph-mon[26617]: [26/Apr/2024:04:49:17] ENGINE Serving on https://172.21.15.12:7150 2024-04-26T04:49:18.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:18 smithi012 ceph-mon[26617]: [26/Apr/2024:04:49:17] ENGINE Bus STARTED 2024-04-26T04:49:18.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:18 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:18.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:18 smithi012 ceph-mon[26617]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-26T04:49:18.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:18 smithi012 ceph-mon[26617]: mgrmap e11: y(active, since 1.0076s) 2024-04-26T04:49:18.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:18 smithi012 ceph-mon[26617]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-26T04:49:18.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:18 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:20.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:19 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:20.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:19 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:20.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:19 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:20.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:19 smithi012 ceph-mon[26617]: mgrmap e12: y(active, since 2s) 2024-04-26T04:49:20.297 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-04-26T04:49:20.298 INFO:teuthology.orchestra.run.smithi012.stdout:Creating initial admin user... 2024-04-26T04:49:21.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:20 smithi012 ceph-mon[26617]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:49:22.131 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$HwLXGuJtklmULgEdYQl7eud7nOBGyO4iWLxCWci5wXREYfADTeH9a", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1714106961, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-04-26T04:49:22.131 INFO:teuthology.orchestra.run.smithi012.stdout:Fetching dashboard port number... 2024-04-26T04:49:22.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:22 smithi012 ceph-mon[26617]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:49:22.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:22 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:23.321 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stdout 8443 2024-04-26T04:49:23.330 INFO:teuthology.orchestra.run.smithi012.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-26T04:49:23.331 INFO:teuthology.orchestra.run.smithi012.stdout:systemctl: stdout disabled 2024-04-26T04:49:23.338 INFO:teuthology.orchestra.run.smithi012.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-26T04:49:23.338 INFO:teuthology.orchestra.run.smithi012.stdout:systemctl: stdout inactive 2024-04-26T04:49:23.338 INFO:teuthology.orchestra.run.smithi012.stdout:firewalld.service is not enabled 2024-04-26T04:49:23.338 INFO:teuthology.orchestra.run.smithi012.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-04-26T04:49:23.339 INFO:teuthology.orchestra.run.smithi012.stdout:Ceph Dashboard is now available at: 2024-04-26T04:49:23.340 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:49:23.340 INFO:teuthology.orchestra.run.smithi012.stdout: URL: https://smithi012:8443/ 2024-04-26T04:49:23.340 INFO:teuthology.orchestra.run.smithi012.stdout: User: admin 2024-04-26T04:49:23.340 INFO:teuthology.orchestra.run.smithi012.stdout: Password: 4sdz37l7q7 2024-04-26T04:49:23.340 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:49:23.340 INFO:teuthology.orchestra.run.smithi012.stdout:Saving cluster configuration to /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config directory 2024-04-26T04:49:23.340 INFO:teuthology.orchestra.run.smithi012.stdout:Enabling autotune for osd_memory_target 2024-04-26T04:49:23.637 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:23 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/3404591588' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-26T04:49:24.802 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:24 smithi012 ceph-mon[26617]: mgrmap e13: y(active, since 6s) 2024-04-26T04:49:25.767 INFO:teuthology.orchestra.run.smithi012.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-04-26T04:49:25.767 INFO:teuthology.orchestra.run.smithi012.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-04-26T04:49:25.768 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:49:25.768 INFO:teuthology.orchestra.run.smithi012.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-04-26T04:49:25.768 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:49:25.768 INFO:teuthology.orchestra.run.smithi012.stdout:Or, if you are only running a single cluster on this host: 2024-04-26T04:49:25.768 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:49:25.768 INFO:teuthology.orchestra.run.smithi012.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-04-26T04:49:25.768 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:49:25.768 INFO:teuthology.orchestra.run.smithi012.stdout:Please consider enabling telemetry to help improve Ceph: 2024-04-26T04:49:25.768 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:49:25.768 INFO:teuthology.orchestra.run.smithi012.stdout: ceph telemetry on 2024-04-26T04:49:25.768 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:49:25.768 INFO:teuthology.orchestra.run.smithi012.stdout:For more information see: 2024-04-26T04:49:25.768 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:49:25.769 INFO:teuthology.orchestra.run.smithi012.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-04-26T04:49:25.769 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:49:25.769 INFO:teuthology.orchestra.run.smithi012.stdout:Bootstrap complete. 2024-04-26T04:49:25.805 INFO:tasks.cephadm:Fetching config... 2024-04-26T04:49:25.805 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-04-26T04:49:25.805 DEBUG:teuthology.orchestra.run.smithi012:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-04-26T04:49:25.821 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-04-26T04:49:25.821 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-04-26T04:49:25.821 DEBUG:teuthology.orchestra.run.smithi012:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-04-26T04:49:25.877 INFO:tasks.cephadm:Fetching mon keyring... 2024-04-26T04:49:25.877 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-04-26T04:49:25.877 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd if=/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/keyring of=/dev/stdout 2024-04-26T04:49:25.942 INFO:tasks.cephadm:Fetching pub ssh key... 2024-04-26T04:49:25.942 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-04-26T04:49:25.942 DEBUG:teuthology.orchestra.run.smithi012:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-04-26T04:49:25.997 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-04-26T04:49:25.997 DEBUG:teuthology.orchestra.run.smithi012:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDThcoSHXia9Nrj2bPpewcabZozpxqaZ0lAcyjLAtHIf7s8kYQUvJcJq/tzhCYL4+gkXG87f8071cQpmKspSmXfYPH4wAkx0IUVhwlTM+UQ8G3SdzcueW3zx+w3uAmgmSgVXK5nbAB8zyUXkEy7uGfigLJtkKfPjUREse5cjfpEdQlz9GvBCerT9Wbdtc9hvSdTtpkVr2vzlYNwsGt9ZF4b5XkSYtxX4g8ZrP84TJIoHo2VAF09llX/CDM+Xmc8zyfixqm+eHuhf7WYqyQZtg7xF5tMjrAmteQo2Wvbo68umDyZtZlr0Y2eXPcaUMqn7yvDN8pup+5feK4B2StVQ0WySKk7HavzBain1ihbDDYvSVQVhClocjt2nyTt5Oqss5/vPzo0hXf9oANPgjqsTW/OnSZHzQkZUDlZC3Q+NvAaaTd9IbwNdZ+RvcxbPHKFOERHNnpOaTXnS59k3cEBPEZz1ZDIC8gz2Mu0fruTO1JD+LzgSr2eVzkIt4INZdjCTu0= ceph-0b9350c6-0388-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-26T04:49:26.071 INFO:teuthology.orchestra.run.smithi012.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDThcoSHXia9Nrj2bPpewcabZozpxqaZ0lAcyjLAtHIf7s8kYQUvJcJq/tzhCYL4+gkXG87f8071cQpmKspSmXfYPH4wAkx0IUVhwlTM+UQ8G3SdzcueW3zx+w3uAmgmSgVXK5nbAB8zyUXkEy7uGfigLJtkKfPjUREse5cjfpEdQlz9GvBCerT9Wbdtc9hvSdTtpkVr2vzlYNwsGt9ZF4b5XkSYtxX4g8ZrP84TJIoHo2VAF09llX/CDM+Xmc8zyfixqm+eHuhf7WYqyQZtg7xF5tMjrAmteQo2Wvbo68umDyZtZlr0Y2eXPcaUMqn7yvDN8pup+5feK4B2StVQ0WySKk7HavzBain1ihbDDYvSVQVhClocjt2nyTt5Oqss5/vPzo0hXf9oANPgjqsTW/OnSZHzQkZUDlZC3Q+NvAaaTd9IbwNdZ+RvcxbPHKFOERHNnpOaTXnS59k3cEBPEZz1ZDIC8gz2Mu0fruTO1JD+LzgSr2eVzkIt4INZdjCTu0= ceph-0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T04:49:26.094 DEBUG:teuthology.orchestra.run.smithi116:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDThcoSHXia9Nrj2bPpewcabZozpxqaZ0lAcyjLAtHIf7s8kYQUvJcJq/tzhCYL4+gkXG87f8071cQpmKspSmXfYPH4wAkx0IUVhwlTM+UQ8G3SdzcueW3zx+w3uAmgmSgVXK5nbAB8zyUXkEy7uGfigLJtkKfPjUREse5cjfpEdQlz9GvBCerT9Wbdtc9hvSdTtpkVr2vzlYNwsGt9ZF4b5XkSYtxX4g8ZrP84TJIoHo2VAF09llX/CDM+Xmc8zyfixqm+eHuhf7WYqyQZtg7xF5tMjrAmteQo2Wvbo68umDyZtZlr0Y2eXPcaUMqn7yvDN8pup+5feK4B2StVQ0WySKk7HavzBain1ihbDDYvSVQVhClocjt2nyTt5Oqss5/vPzo0hXf9oANPgjqsTW/OnSZHzQkZUDlZC3Q+NvAaaTd9IbwNdZ+RvcxbPHKFOERHNnpOaTXnS59k3cEBPEZz1ZDIC8gz2Mu0fruTO1JD+LzgSr2eVzkIt4INZdjCTu0= ceph-0b9350c6-0388-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-26T04:49:26.132 INFO:teuthology.orchestra.run.smithi116.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDThcoSHXia9Nrj2bPpewcabZozpxqaZ0lAcyjLAtHIf7s8kYQUvJcJq/tzhCYL4+gkXG87f8071cQpmKspSmXfYPH4wAkx0IUVhwlTM+UQ8G3SdzcueW3zx+w3uAmgmSgVXK5nbAB8zyUXkEy7uGfigLJtkKfPjUREse5cjfpEdQlz9GvBCerT9Wbdtc9hvSdTtpkVr2vzlYNwsGt9ZF4b5XkSYtxX4g8ZrP84TJIoHo2VAF09llX/CDM+Xmc8zyfixqm+eHuhf7WYqyQZtg7xF5tMjrAmteQo2Wvbo68umDyZtZlr0Y2eXPcaUMqn7yvDN8pup+5feK4B2StVQ0WySKk7HavzBain1ihbDDYvSVQVhClocjt2nyTt5Oqss5/vPzo0hXf9oANPgjqsTW/OnSZHzQkZUDlZC3Q+NvAaaTd9IbwNdZ+RvcxbPHKFOERHNnpOaTXnS59k3cEBPEZz1ZDIC8gz2Mu0fruTO1JD+LzgSr2eVzkIt4INZdjCTu0= ceph-0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T04:49:26.144 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-04-26T04:49:26.406 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:49:26.512 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:26 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/679173249' entity='client.admin' 2024-04-26T04:49:29.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:28 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:29.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:28 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:29.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:28 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi012", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:49:29.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:28 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:29.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:28 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:29.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:28 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:29.299 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-04-26T04:49:29.299 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-04-26T04:49:29.565 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:49:30.002 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:29 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/2318350269' entity='client.admin' 2024-04-26T04:49:30.002 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:29 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:30.002 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:29 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:32.303 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi116 2024-04-26T04:49:32.303 DEBUG:teuthology.orchestra.run.smithi116:> set -ex 2024-04-26T04:49:32.304 DEBUG:teuthology.orchestra.run.smithi116:> dd of=/etc/ceph/ceph.conf 2024-04-26T04:49:32.322 DEBUG:teuthology.orchestra.run.smithi116:> set -ex 2024-04-26T04:49:32.322 DEBUG:teuthology.orchestra.run.smithi116:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:49:32.380 INFO:tasks.cephadm:Adding host smithi116 to orchestrator... 2024-04-26T04:49:32.380 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph orch host add smithi116 2024-04-26T04:49:32.512 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:32 smithi012 ceph-mon[26617]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:49:32.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:32 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:32.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:32 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:32.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:32 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:32.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:32 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:49:32.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:32 smithi012 ceph-mon[26617]: Updating smithi012:/etc/ceph/ceph.conf 2024-04-26T04:49:32.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:32 smithi012 ceph-mon[26617]: Updating smithi012:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:49:32.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:32 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:32.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:32 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:32.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:32 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:32.635 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:49:33.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:33 smithi012 ceph-mon[26617]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:49:33.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:33 smithi012 ceph-mon[26617]: Updating smithi012:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:49:35.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:34 smithi012 ceph-mon[26617]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi116", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:49:36.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:35 smithi012 ceph-mon[26617]: Deploying cephadm binary to smithi116 2024-04-26T04:49:36.263 INFO:teuthology.orchestra.run.smithi012.stdout:Added host 'smithi116' with addr '172.21.15.116' 2024-04-26T04:49:36.853 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph orch host ls --format=json 2024-04-26T04:49:37.113 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:49:37.511 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:37 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:37.511 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:37 smithi012 ceph-mon[26617]: Added host smithi116 2024-04-26T04:49:37.511 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:37 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:37.511 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:37 smithi012 ceph-mon[26617]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:49:37.511 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:37 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:37.511 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:37 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:38.724 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:49:38.724 INFO:teuthology.orchestra.run.smithi012.stdout:[{"addr": "172.21.15.12", "hostname": "smithi012", "labels": [], "status": ""}, {"addr": "172.21.15.116", "hostname": "smithi116", "labels": [], "status": ""}] 2024-04-26T04:49:39.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:38 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:39.299 INFO:tasks.cephadm:Setting crush tunables to default 2024-04-26T04:49:39.299 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph osd crush tunables default 2024-04-26T04:49:39.558 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:49:39.940 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:39 smithi012 ceph-mon[26617]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:49:39.940 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:39 smithi012 ceph-mon[26617]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:49:41.530 INFO:teuthology.orchestra.run.smithi012.stderr:adjusted tunables profile to default 2024-04-26T04:49:41.673 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:41 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:41.673 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:41 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:41.673 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:41 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:41.673 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:41 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:41.674 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:41 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi116", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:49:41.674 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:41 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:41.674 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:41 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:49:41.674 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:41 smithi012 ceph-mon[26617]: Updating smithi116:/etc/ceph/ceph.conf 2024-04-26T04:49:41.674 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:41 smithi012 ceph-mon[26617]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:49:41.674 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:41 smithi012 ceph-mon[26617]: Updating smithi116:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:49:41.674 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:41 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/2612484146' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-26T04:49:41.674 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:41 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:41.674 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:41 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:41.674 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:41 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:42.085 INFO:tasks.cephadm:Adding mon.a on smithi012 2024-04-26T04:49:42.085 INFO:tasks.cephadm:Adding mon.c on smithi012 2024-04-26T04:49:42.085 INFO:tasks.cephadm:Adding mon.b on smithi116 2024-04-26T04:49:42.085 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph orch apply mon '3;smithi012:172.21.15.12=a;smithi012:[v2:172.21.15.12:3301,v1:172.21.15.12:6790]=c;smithi116:172.21.15.116=b' 2024-04-26T04:49:42.320 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:49:43.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:42 smithi012 ceph-mon[26617]: Updating smithi116:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:49:43.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:42 smithi012 ceph-mon[26617]: Updating smithi116:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:49:43.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:42 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/2612484146' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-26T04:49:43.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:42 smithi012 ceph-mon[26617]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:49:43.959 INFO:teuthology.orchestra.run.smithi116.stdout:Scheduled mon update... 2024-04-26T04:49:44.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:43 smithi012 ceph-mon[26617]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:49:44.587 DEBUG:teuthology.orchestra.run.smithi012:mon.c> sudo journalctl -f -n 0 -u ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mon.c.service 2024-04-26T04:49:44.590 DEBUG:teuthology.orchestra.run.smithi116:mon.b> sudo journalctl -f -n 0 -u ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mon.b.service 2024-04-26T04:49:44.592 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-26T04:49:44.592 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-26T04:49:45.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:44 smithi012 ceph-mon[26617]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi012:172.21.15.12=a;smithi012:[v2:172.21.15.12:3301,v1:172.21.15.12:6790]=c;smithi116:172.21.15.116=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:49:45.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:44 smithi012 ceph-mon[26617]: Saving service mon spec with placement smithi012:172.21.15.12=a;smithi012:[v2:172.21.15.12:3301,v1:172.21.15.12:6790]=c;smithi116:172.21.15.116=b;count:3 2024-04-26T04:49:45.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:44 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:45.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:44 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:45.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:44 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:45.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:44 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:49:45.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:44 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:45.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:44 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:49:45.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:44 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:45.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:44 smithi012 ceph-mon[26617]: Deploying daemon mon.b on smithi116 2024-04-26T04:49:45.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:44 smithi012 ceph-mon[26617]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:49:46.848 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:49:47.899 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:47 smithi116 systemd[1]: Starting Ceph mon.b for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:49:48.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:47 smithi012 ceph-mon[26617]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:49:48.677 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:49:48.677 INFO:teuthology.orchestra.run.smithi116.stdout:{"epoch":1,"fsid":"0b9350c6-0388-11ef-bc93-c7b262605968","modified":"2024-04-26T04:48:15.190556Z","created":"2024-04-26T04:48:15.190556Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:3300","nonce":0},{"type":"v1","addr":"172.21.15.12:6789","nonce":0}]},"addr":"172.21.15.12:6789/0","public_addr":"172.21.15.12:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-26T04:49:48.678 INFO:teuthology.orchestra.run.smithi116.stderr:dumped monmap epoch 1 2024-04-26T04:49:48.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 podman[29858]: 2024-04-26 04:49:48.38410157 +0000 UTC m=+0.104083813 image pull 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17 2024-04-26T04:49:48.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 podman[29858]: 2024-04-26 04:49:48.486230595 +0000 UTC m=+0.206212889 container create e6cd16f014178dae87b9d89437299336c95137420d1ea34ca296a9beeede4960 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-b, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, RELEASE=quincy-c61472c) 2024-04-26T04:49:48.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 podman[29858]: 2024-04-26 04:49:48.663361524 +0000 UTC m=+0.383343767 container init e6cd16f014178dae87b9d89437299336c95137420d1ea34ca296a9beeede4960 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-b, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=quincy-c61472c, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-04-26T04:49:48.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 podman[29858]: 2024-04-26 04:49:48.66632063 +0000 UTC m=+0.386302873 container start e6cd16f014178dae87b9d89437299336c95137420d1ea34ca296a9beeede4960 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-b, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-c61472c, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS) 2024-04-26T04:49:48.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: set uid:gid to 167:167 (ceph:ceph) 2024-04-26T04:49:48.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable), process ceph-mon, pid 2 2024-04-26T04:49:48.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: pidfile_write: ignore empty --pid-file 2024-04-26T04:49:48.743 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: load: jerasure load: lrc 2024-04-26T04:49:48.743 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: RocksDB version: 6.15.5 2024-04-26T04:49:48.743 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-26T04:49:48.743 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Compile date Apr 24 2024 2024-04-26T04:49:48.743 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: DB SUMMARY 2024-04-26T04:49:48.743 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: DB Session ID: ODVHHASVLXX2G0YSEV4M 2024-04-26T04:49:48.743 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: CURRENT file: CURRENT 2024-04-26T04:49:48.743 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: IDENTITY file: IDENTITY 2024-04-26T04:49:48.743 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-04-26T04:49:48.743 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-04-26T04:49:48.743 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 511 ; 2024-04-26T04:49:48.743 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.error_if_exists: 0 2024-04-26T04:49:48.743 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.create_if_missing: 0 2024-04-26T04:49:48.744 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.paranoid_checks: 1 2024-04-26T04:49:48.744 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-26T04:49:48.744 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.env: 0x559531051b00 2024-04-26T04:49:48.744 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.fs: Posix File System 2024-04-26T04:49:48.744 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.info_log: 0x559533052ee0 2024-04-26T04:49:48.744 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.max_file_opening_threads: 16 2024-04-26T04:49:48.744 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.statistics: (nil) 2024-04-26T04:49:48.744 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.use_fsync: 0 2024-04-26T04:49:48.744 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.max_log_file_size: 0 2024-04-26T04:49:48.744 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-26T04:49:48.744 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-26T04:49:48.744 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.keep_log_file_num: 1000 2024-04-26T04:49:48.744 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.recycle_log_file_num: 0 2024-04-26T04:49:48.744 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.allow_fallocate: 1 2024-04-26T04:49:48.745 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.allow_mmap_reads: 0 2024-04-26T04:49:48.745 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.allow_mmap_writes: 0 2024-04-26T04:49:48.745 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.use_direct_reads: 0 2024-04-26T04:49:48.745 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T04:49:48.745 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T04:49:48.745 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.db_log_dir: 2024-04-26T04:49:48.745 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2024-04-26T04:49:48.745 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T04:49:48.745 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T04:49:48.745 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T04:49:48.745 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T04:49:48.745 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T04:49:48.745 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T04:49:48.746 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T04:49:48.746 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T04:49:48.746 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.write_buffer_manager: 0x559533dc2990 2024-04-26T04:49:48.746 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T04:49:48.746 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-26T04:49:48.746 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T04:49:48.746 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T04:49:48.746 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T04:49:48.746 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T04:49:48.746 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T04:49:48.748 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T04:49:48.748 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T04:49:48.748 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.unordered_write: 0 2024-04-26T04:49:48.748 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T04:49:48.748 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T04:49:48.748 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T04:49:48.748 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T04:49:48.748 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.row_cache: None 2024-04-26T04:49:48.749 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.wal_filter: None 2024-04-26T04:49:48.749 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T04:49:48.749 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T04:49:48.749 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.preserve_deletes: 0 2024-04-26T04:49:48.749 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.two_write_queues: 0 2024-04-26T04:49:48.749 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T04:49:48.749 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.atomic_flush: 0 2024-04-26T04:49:48.749 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T04:49:48.749 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T04:49:48.749 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T04:49:48.749 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.log_readahead_size: 0 2024-04-26T04:49:48.749 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T04:49:48.749 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T04:49:48.750 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T04:49:48.750 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T04:49:48.750 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T04:49:48.750 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T04:49:48.750 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.max_background_jobs: 2 2024-04-26T04:49:48.750 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.max_background_compactions: -1 2024-04-26T04:49:48.750 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.max_subcompactions: 1 2024-04-26T04:49:48.750 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T04:49:48.750 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T04:49:48.750 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T04:49:48.750 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T04:49:48.750 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T04:49:48.750 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T04:49:48.751 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T04:49:48.751 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T04:49:48.751 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.max_open_files: -1 2024-04-26T04:49:48.751 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T04:49:48.751 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T04:49:48.751 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T04:49:48.751 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T04:49:48.751 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.max_background_flushes: -1 2024-04-26T04:49:48.751 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Compression algorithms supported: 2024-04-26T04:49:48.751 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T04:49:48.751 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: kZSTD supported: 0 2024-04-26T04:49:48.751 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: kXpressCompression supported: 0 2024-04-26T04:49:48.751 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T04:49:48.751 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: kZlibCompression supported: 1 2024-04-26T04:49:48.752 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: kSnappyCompression supported: 1 2024-04-26T04:49:48.752 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: kLZ4Compression supported: 1 2024-04-26T04:49:48.752 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: kBZip2Compression supported: 0 2024-04-26T04:49:48.752 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T04:49:48.752 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2024-04-26T04:49:48.752 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-04-26T04:49:48.752 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T04:49:48.752 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.merge_operator: 2024-04-26T04:49:48.752 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.compaction_filter: None 2024-04-26T04:49:48.752 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T04:49:48.752 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T04:49:48.752 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T04:49:48.752 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T04:49:48.753 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5595330a2f58) 2024-04-26T04:49:48.753 INFO:journalctl@ceph.mon.b.smithi116.stdout: cache_index_and_filter_blocks: 1 2024-04-26T04:49:48.753 INFO:journalctl@ceph.mon.b.smithi116.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T04:49:48.753 INFO:journalctl@ceph.mon.b.smithi116.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T04:49:48.753 INFO:journalctl@ceph.mon.b.smithi116.stdout: pin_top_level_index_and_filter: 1 2024-04-26T04:49:48.753 INFO:journalctl@ceph.mon.b.smithi116.stdout: index_type: 0 2024-04-26T04:49:48.753 INFO:journalctl@ceph.mon.b.smithi116.stdout: data_block_index_type: 0 2024-04-26T04:49:48.753 INFO:journalctl@ceph.mon.b.smithi116.stdout: index_shortening: 1 2024-04-26T04:49:48.753 INFO:journalctl@ceph.mon.b.smithi116.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T04:49:48.753 INFO:journalctl@ceph.mon.b.smithi116.stdout: hash_index_allow_collision: 1 2024-04-26T04:49:48.753 INFO:journalctl@ceph.mon.b.smithi116.stdout: checksum: 1 2024-04-26T04:49:48.753 INFO:journalctl@ceph.mon.b.smithi116.stdout: no_block_cache: 0 2024-04-26T04:49:48.753 INFO:journalctl@ceph.mon.b.smithi116.stdout: block_cache: 0x5595330d5090 2024-04-26T04:49:48.753 INFO:journalctl@ceph.mon.b.smithi116.stdout: block_cache_name: BinnedLRUCache 2024-04-26T04:49:48.754 INFO:journalctl@ceph.mon.b.smithi116.stdout: block_cache_options: 2024-04-26T04:49:48.754 INFO:journalctl@ceph.mon.b.smithi116.stdout: capacity : 536870912 2024-04-26T04:49:48.754 INFO:journalctl@ceph.mon.b.smithi116.stdout: num_shard_bits : 4 2024-04-26T04:49:48.754 INFO:journalctl@ceph.mon.b.smithi116.stdout: strict_capacity_limit : 0 2024-04-26T04:49:48.754 INFO:journalctl@ceph.mon.b.smithi116.stdout: high_pri_pool_ratio: 0.000 2024-04-26T04:49:48.754 INFO:journalctl@ceph.mon.b.smithi116.stdout: block_cache_compressed: (nil) 2024-04-26T04:49:48.754 INFO:journalctl@ceph.mon.b.smithi116.stdout: persistent_cache: (nil) 2024-04-26T04:49:48.754 INFO:journalctl@ceph.mon.b.smithi116.stdout: block_size: 4096 2024-04-26T04:49:48.754 INFO:journalctl@ceph.mon.b.smithi116.stdout: block_size_deviation: 10 2024-04-26T04:49:48.754 INFO:journalctl@ceph.mon.b.smithi116.stdout: block_restart_interval: 16 2024-04-26T04:49:48.754 INFO:journalctl@ceph.mon.b.smithi116.stdout: index_block_restart_interval: 1 2024-04-26T04:49:48.754 INFO:journalctl@ceph.mon.b.smithi116.stdout: metadata_block_size: 4096 2024-04-26T04:49:48.755 INFO:journalctl@ceph.mon.b.smithi116.stdout: partition_filters: 0 2024-04-26T04:49:48.755 INFO:journalctl@ceph.mon.b.smithi116.stdout: use_delta_encoding: 1 2024-04-26T04:49:48.755 INFO:journalctl@ceph.mon.b.smithi116.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-26T04:49:48.755 INFO:journalctl@ceph.mon.b.smithi116.stdout: whole_key_filtering: 1 2024-04-26T04:49:48.755 INFO:journalctl@ceph.mon.b.smithi116.stdout: verify_compression: 0 2024-04-26T04:49:48.755 INFO:journalctl@ceph.mon.b.smithi116.stdout: read_amp_bytes_per_bit: 0 2024-04-26T04:49:48.755 INFO:journalctl@ceph.mon.b.smithi116.stdout: format_version: 4 2024-04-26T04:49:48.755 INFO:journalctl@ceph.mon.b.smithi116.stdout: enable_index_compression: 1 2024-04-26T04:49:48.755 INFO:journalctl@ceph.mon.b.smithi116.stdout: block_align: 0 2024-04-26T04:49:48.755 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T04:49:48.755 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T04:49:48.756 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.compression: NoCompression 2024-04-26T04:49:48.756 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T04:49:48.756 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T04:49:48.756 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T04:49:48.756 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.num_levels: 7 2024-04-26T04:49:48.756 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T04:49:48.756 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T04:49:48.756 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T04:49:48.756 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T04:49:48.756 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T04:49:48.756 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T04:49:48.756 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T04:49:48.757 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:49:48.757 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T04:49:48.757 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T04:49:48.757 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T04:49:48.757 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T04:49:48.757 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T04:49:48.757 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T04:49:48.757 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:49:48.758 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T04:49:48.758 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T04:49:48.758 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T04:49:48.758 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T04:49:48.758 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T04:49:48.758 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T04:49:48.758 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T04:49:48.759 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T04:49:48.759 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T04:49:48.759 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T04:49:48.759 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T04:49:48.759 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T04:49:48.759 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T04:49:48.759 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T04:49:48.759 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T04:49:48.759 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T04:49:48.759 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T04:49:48.759 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T04:49:48.759 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T04:49:48.759 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.arena_block_size: 4194304 2024-04-26T04:49:48.759 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T04:49:48.760 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T04:49:48.760 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-26T04:49:48.760 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T04:49:48.760 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T04:49:48.760 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T04:49:48.760 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T04:49:48.760 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T04:49:48.760 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T04:49:48.760 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T04:49:48.760 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T04:49:48.760 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T04:49:48.760 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T04:49:48.761 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T04:49:48.761 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.table_properties_collectors: 2024-04-26T04:49:48.761 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.inplace_update_support: 0 2024-04-26T04:49:48.761 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T04:49:48.761 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T04:49:48.761 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T04:49:48.761 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T04:49:48.761 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.bloom_locality: 0 2024-04-26T04:49:48.761 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.max_successive_merges: 0 2024-04-26T04:49:48.761 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T04:49:48.761 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T04:49:48.761 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T04:49:48.761 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T04:49:48.761 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.ttl: 2592000 2024-04-26T04:49:48.761 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T04:49:48.762 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.enable_blob_files: false 2024-04-26T04:49:48.762 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.min_blob_size: 0 2024-04-26T04:49:48.762 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T04:49:48.762 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T04:49:48.762 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T04:49:48.762 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T04:49:48.762 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T04:49:48.762 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-04-26T04:49:48.762 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-04-26T04:49:48.762 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106988693965, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-04-26T04:49:48.762 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-04-26T04:49:48.763 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-26T04:49:48.763 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106988694276, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1714106988, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "1bb3f39f-47da-4dfd-be5b-6e12535c1a94", "db_session_id": "ODVHHASVLXX2G0YSEV4M"}} 2024-04-26T04:49:48.763 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-04-26T04:49:48.763 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106988694590, "job": 1, "event": "recovery_finished"} 2024-04-26T04:49:48.763 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T04:49:48.763 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x559533069180 2024-04-26T04:49:48.763 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: DB pointer 0x55953313e000 2024-04-26T04:49:48.763 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-04-26T04:49:48.763 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-26T04:49:48.763 INFO:journalctl@ceph.mon.b.smithi116.stdout: ** DB Stats ** 2024-04-26T04:49:48.763 INFO:journalctl@ceph.mon.b.smithi116.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:49:48.763 INFO:journalctl@ceph.mon.b.smithi116.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T04:49:48.763 INFO:journalctl@ceph.mon.b.smithi116.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T04:49:48.763 INFO:journalctl@ceph.mon.b.smithi116.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:49:48.764 INFO:journalctl@ceph.mon.b.smithi116.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T04:49:48.764 INFO:journalctl@ceph.mon.b.smithi116.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-26T04:49:48.764 INFO:journalctl@ceph.mon.b.smithi116.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:49:48.764 INFO:journalctl@ceph.mon.b.smithi116.stdout: 2024-04-26T04:49:48.764 INFO:journalctl@ceph.mon.b.smithi116.stdout: ** Compaction Stats [default] ** 2024-04-26T04:49:48.764 INFO:journalctl@ceph.mon.b.smithi116.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:49:48.764 INFO:journalctl@ceph.mon.b.smithi116.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:49:48.764 INFO:journalctl@ceph.mon.b.smithi116.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.9 0.00 0.00 1 0.000 0 0 2024-04-26T04:49:48.764 INFO:journalctl@ceph.mon.b.smithi116.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.9 0.00 0.00 1 0.000 0 0 2024-04-26T04:49:48.764 INFO:journalctl@ceph.mon.b.smithi116.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.9 0.00 0.00 1 0.000 0 0 2024-04-26T04:49:48.764 INFO:journalctl@ceph.mon.b.smithi116.stdout: 2024-04-26T04:49:48.764 INFO:journalctl@ceph.mon.b.smithi116.stdout: ** Compaction Stats [default] ** 2024-04-26T04:49:48.764 INFO:journalctl@ceph.mon.b.smithi116.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:49:48.765 INFO:journalctl@ceph.mon.b.smithi116.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:49:48.765 INFO:journalctl@ceph.mon.b.smithi116.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.9 0.00 0.00 1 0.000 0 0 2024-04-26T04:49:48.765 INFO:journalctl@ceph.mon.b.smithi116.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:49:48.765 INFO:journalctl@ceph.mon.b.smithi116.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-26T04:49:48.765 INFO:journalctl@ceph.mon.b.smithi116.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T04:49:48.765 INFO:journalctl@ceph.mon.b.smithi116.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T04:49:48.765 INFO:journalctl@ceph.mon.b.smithi116.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T04:49:48.765 INFO:journalctl@ceph.mon.b.smithi116.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T04:49:48.765 INFO:journalctl@ceph.mon.b.smithi116.stdout: Cumulative compaction: 0.00 GB write, 0.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:49:48.765 INFO:journalctl@ceph.mon.b.smithi116.stdout: Interval compaction: 0.00 GB write, 0.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:49:48.765 INFO:journalctl@ceph.mon.b.smithi116.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T04:49:48.765 INFO:journalctl@ceph.mon.b.smithi116.stdout: 2024-04-26T04:49:48.765 INFO:journalctl@ceph.mon.b.smithi116.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T04:49:48.766 INFO:journalctl@ceph.mon.b.smithi116.stdout: 2024-04-26T04:49:48.766 INFO:journalctl@ceph.mon.b.smithi116.stdout: ** Compaction Stats [default] ** 2024-04-26T04:49:48.766 INFO:journalctl@ceph.mon.b.smithi116.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:49:48.766 INFO:journalctl@ceph.mon.b.smithi116.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:49:48.766 INFO:journalctl@ceph.mon.b.smithi116.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.9 0.00 0.00 1 0.000 0 0 2024-04-26T04:49:48.766 INFO:journalctl@ceph.mon.b.smithi116.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.9 0.00 0.00 1 0.000 0 0 2024-04-26T04:49:48.766 INFO:journalctl@ceph.mon.b.smithi116.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-26T04:49:48.766 INFO:journalctl@ceph.mon.b.smithi116.stdout: 2024-04-26T04:49:48.766 INFO:journalctl@ceph.mon.b.smithi116.stdout: ** Compaction Stats [default] ** 2024-04-26T04:49:48.766 INFO:journalctl@ceph.mon.b.smithi116.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:49:48.766 INFO:journalctl@ceph.mon.b.smithi116.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:49:48.766 INFO:journalctl@ceph.mon.b.smithi116.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.9 0.00 0.00 1 0.000 0 0 2024-04-26T04:49:48.767 INFO:journalctl@ceph.mon.b.smithi116.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:49:48.767 INFO:journalctl@ceph.mon.b.smithi116.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-26T04:49:48.767 INFO:journalctl@ceph.mon.b.smithi116.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T04:49:48.767 INFO:journalctl@ceph.mon.b.smithi116.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T04:49:48.767 INFO:journalctl@ceph.mon.b.smithi116.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T04:49:48.767 INFO:journalctl@ceph.mon.b.smithi116.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T04:49:48.767 INFO:journalctl@ceph.mon.b.smithi116.stdout: Cumulative compaction: 0.00 GB write, 0.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:49:48.767 INFO:journalctl@ceph.mon.b.smithi116.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:49:48.767 INFO:journalctl@ceph.mon.b.smithi116.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T04:49:48.767 INFO:journalctl@ceph.mon.b.smithi116.stdout: 2024-04-26T04:49:48.767 INFO:journalctl@ceph.mon.b.smithi116.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T04:49:48.768 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: mon.b does not exist in monmap, will attempt to join an existing cluster 2024-04-26T04:49:48.768 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: using public_addr v2:172.21.15.116:0/0 -> [v2:172.21.15.116:3300/0,v1:172.21.15.116:6789/0] 2024-04-26T04:49:48.768 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: starting mon.b rank -1 at public addrs [v2:172.21.15.116:3300/0,v1:172.21.15.116:6789/0] at bind addrs [v2:172.21.15.116:3300/0,v1:172.21.15.116:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T04:49:48.768 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: mon.b@-1(???) e0 preinit fsid 0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T04:49:48.768 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: mon.b@-1(synchronizing).mds e1 new map 2024-04-26T04:49:48.768 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: mon.b@-1(synchronizing).mds e1 print_map 2024-04-26T04:49:48.768 INFO:journalctl@ceph.mon.b.smithi116.stdout: e1 2024-04-26T04:49:48.768 INFO:journalctl@ceph.mon.b.smithi116.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T04:49:48.768 INFO:journalctl@ceph.mon.b.smithi116.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T04:49:48.768 INFO:journalctl@ceph.mon.b.smithi116.stdout: legacy client fscid: -1 2024-04-26T04:49:48.768 INFO:journalctl@ceph.mon.b.smithi116.stdout: 2024-04-26T04:49:48.768 INFO:journalctl@ceph.mon.b.smithi116.stdout: No filesystems configured 2024-04-26T04:49:48.769 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-04-26T04:49:48.769 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-04-26T04:49:48.769 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-04-26T04:49:48.769 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-04-26T04:49:48.769 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-04-26T04:49:48.769 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-04-26T04:49:48.769 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-26T04:49:48.769 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:49:48.769 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:49:48.769 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:49:48.770 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: Updating smithi116:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:49:48.770 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: Updating smithi116:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:49:48.770 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/2612484146' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-26T04:49:48.770 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:49:48.770 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:49:48.770 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi012:172.21.15.12=a;smithi012:[v2:172.21.15.12:3301,v1:172.21.15.12:6790]=c;smithi116:172.21.15.116=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:49:48.770 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: Saving service mon spec with placement smithi012:172.21.15.12=a;smithi012:[v2:172.21.15.12:3301,v1:172.21.15.12:6790]=c;smithi116:172.21.15.116=b;count:3 2024-04-26T04:49:48.770 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:48.770 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:48.770 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:48.770 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:49:48.770 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:48.770 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:49:48.770 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:48.771 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: Deploying daemon mon.b on smithi116 2024-04-26T04:49:48.771 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:49:48.771 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:49:48.771 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 ceph-mon[29897]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-04-26T04:49:48.771 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 bash[29858]: e6cd16f014178dae87b9d89437299336c95137420d1ea34ca296a9beeede4960 2024-04-26T04:49:48.771 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:48 smithi116 systemd[1]: Started Ceph mon.b for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:49:50.400 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-26T04:49:50.400 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-26T04:49:50.656 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:49:54.217 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:53 smithi012 ceph-mon[26617]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:49:54.218 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:53 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:49:54.218 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:53 smithi012 ceph-mon[26617]: mon.a calling monitor election 2024-04-26T04:49:54.218 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:53 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:49:54.218 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:53 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:49:54.218 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:53 smithi012 ceph-mon[26617]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:49:54.218 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:53 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:49:54.218 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:53 smithi012 ceph-mon[26617]: mon.b calling monitor election 2024-04-26T04:49:54.218 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:53 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:49:54.218 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:53 smithi012 ceph-mon[26617]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:49:54.218 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:53 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:49:54.218 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:53 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:49:54.218 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:53 smithi012 ceph-mon[26617]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-26T04:49:54.219 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:53 smithi012 ceph-mon[26617]: monmap e2: 2 mons at {a=[v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0],b=[v2:172.21.15.116:3300/0,v1:172.21.15.116:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:49:54.219 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:53 smithi012 ceph-mon[26617]: fsmap 2024-04-26T04:49:54.219 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:53 smithi012 ceph-mon[26617]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:49:54.219 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:53 smithi012 ceph-mon[26617]: mgrmap e13: y(active, since 37s) 2024-04-26T04:49:54.219 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:53 smithi012 ceph-mon[26617]: overall HEALTH_OK 2024-04-26T04:49:54.219 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:53 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:54.219 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:53 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:54.219 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:53 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:54.219 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:53 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:49:54.219 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:53 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:54.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:53 smithi116 ceph-mon[29897]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:49:54.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:53 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:49:54.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:53 smithi116 ceph-mon[29897]: mon.a calling monitor election 2024-04-26T04:49:54.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:53 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:49:54.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:53 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:49:54.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:53 smithi116 ceph-mon[29897]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:49:54.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:53 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:49:54.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:53 smithi116 ceph-mon[29897]: mon.b calling monitor election 2024-04-26T04:49:54.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:53 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:49:54.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:53 smithi116 ceph-mon[29897]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:49:54.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:53 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:49:54.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:53 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:49:54.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:53 smithi116 ceph-mon[29897]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-26T04:49:54.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:53 smithi116 ceph-mon[29897]: monmap e2: 2 mons at {a=[v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0],b=[v2:172.21.15.116:3300/0,v1:172.21.15.116:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:49:54.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:53 smithi116 ceph-mon[29897]: fsmap 2024-04-26T04:49:54.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:53 smithi116 ceph-mon[29897]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:49:54.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:53 smithi116 ceph-mon[29897]: mgrmap e13: y(active, since 37s) 2024-04-26T04:49:54.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:53 smithi116 ceph-mon[29897]: overall HEALTH_OK 2024-04-26T04:49:54.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:53 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:54.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:53 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:54.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:53 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:54.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:53 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:49:54.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:53 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:54.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:54 smithi116 ceph-mon[29897]: Deploying daemon mon.c on smithi012 2024-04-26T04:49:54.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:54 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:49:55.131 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:54 smithi012 ceph-mon[26617]: Deploying daemon mon.c on smithi012 2024-04-26T04:49:55.131 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:54 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:49:55.475 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:49:55.476 INFO:teuthology.orchestra.run.smithi116.stdout:{"epoch":2,"fsid":"0b9350c6-0388-11ef-bc93-c7b262605968","modified":"2024-04-26T04:49:48.717217Z","created":"2024-04-26T04:48:15.190556Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:3300","nonce":0},{"type":"v1","addr":"172.21.15.12:6789","nonce":0}]},"addr":"172.21.15.12:6789/0","public_addr":"172.21.15.12:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:3300","nonce":0},{"type":"v1","addr":"172.21.15.116:6789","nonce":0}]},"addr":"172.21.15.116:6789/0","public_addr":"172.21.15.116:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-26T04:49:55.478 INFO:teuthology.orchestra.run.smithi116.stderr:dumped monmap epoch 2 2024-04-26T04:49:56.046 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:55 smithi116 ceph-mon[29897]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:49:56.046 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:49:55 smithi116 ceph-mon[29897]: from='client.? 172.21.15.116:0/3214300913' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T04:49:56.079 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:55 smithi012 ceph-mon[26617]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:49:56.080 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:49:55 smithi012 ceph-mon[26617]: from='client.? 172.21.15.116:0/3214300913' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T04:49:56.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 podman[37548]: 2024-04-26 04:49:56.575401026 +0000 UTC m=+0.018563077 image pull 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17 2024-04-26T04:49:57.070 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-26T04:49:57.070 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-26T04:49:57.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 podman[37548]: 2024-04-26 04:49:56.900019664 +0000 UTC m=+0.343181701 container init 0f03d4c16528513e9afcd9c7cf63732f1f25d5bb1bcb3ff09079b69b6a66d946 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-c, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-c61472c, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS) 2024-04-26T04:49:57.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 podman[37548]: 2024-04-26 04:49:56.904245681 +0000 UTC m=+0.347407726 container start 0f03d4c16528513e9afcd9c7cf63732f1f25d5bb1bcb3ff09079b69b6a66d946 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-c, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, RELEASE=quincy-c61472c, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T04:49:57.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: set uid:gid to 167:167 (ceph:ceph) 2024-04-26T04:49:57.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable), process ceph-mon, pid 2 2024-04-26T04:49:57.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: pidfile_write: ignore empty --pid-file 2024-04-26T04:49:57.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: load: jerasure load: lrc 2024-04-26T04:49:57.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: RocksDB version: 6.15.5 2024-04-26T04:49:57.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-26T04:49:57.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Compile date Apr 24 2024 2024-04-26T04:49:57.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: DB SUMMARY 2024-04-26T04:49:57.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: DB Session ID: GU20K0DWKZGU0ZQQ5U29 2024-04-26T04:49:57.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: CURRENT file: CURRENT 2024-04-26T04:49:57.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: IDENTITY file: IDENTITY 2024-04-26T04:49:57.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-04-26T04:49:57.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-04-26T04:49:57.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2024-04-26T04:49:57.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.error_if_exists: 0 2024-04-26T04:49:57.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.create_if_missing: 0 2024-04-26T04:49:57.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.paranoid_checks: 1 2024-04-26T04:49:57.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-26T04:49:57.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.env: 0x5560bbd26b00 2024-04-26T04:49:57.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.fs: Posix File System 2024-04-26T04:49:57.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.info_log: 0x5560bcb6cee0 2024-04-26T04:49:57.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.max_file_opening_threads: 16 2024-04-26T04:49:57.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.statistics: (nil) 2024-04-26T04:49:57.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.use_fsync: 0 2024-04-26T04:49:57.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.max_log_file_size: 0 2024-04-26T04:49:57.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-26T04:49:57.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-26T04:49:57.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.keep_log_file_num: 1000 2024-04-26T04:49:57.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.recycle_log_file_num: 0 2024-04-26T04:49:57.266 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.allow_fallocate: 1 2024-04-26T04:49:57.266 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.allow_mmap_reads: 0 2024-04-26T04:49:57.266 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.allow_mmap_writes: 0 2024-04-26T04:49:57.266 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.use_direct_reads: 0 2024-04-26T04:49:57.266 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T04:49:57.266 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T04:49:57.266 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.db_log_dir: 2024-04-26T04:49:57.266 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2024-04-26T04:49:57.266 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T04:49:57.266 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T04:49:57.266 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T04:49:57.266 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T04:49:57.266 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T04:49:57.267 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T04:49:57.267 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T04:49:57.267 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T04:49:57.267 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.write_buffer_manager: 0x5560bd8dc960 2024-04-26T04:49:57.267 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T04:49:57.267 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-26T04:49:57.267 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T04:49:57.267 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T04:49:57.267 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T04:49:57.267 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T04:49:57.267 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T04:49:57.267 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T04:49:57.267 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T04:49:57.268 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.unordered_write: 0 2024-04-26T04:49:57.268 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T04:49:57.268 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T04:49:57.268 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T04:49:57.268 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T04:49:57.268 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.row_cache: None 2024-04-26T04:49:57.268 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.wal_filter: None 2024-04-26T04:49:57.268 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T04:49:57.269 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T04:49:57.269 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.preserve_deletes: 0 2024-04-26T04:49:57.270 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.two_write_queues: 0 2024-04-26T04:49:57.270 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T04:49:57.270 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.atomic_flush: 0 2024-04-26T04:49:57.270 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T04:49:57.270 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T04:49:57.270 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T04:49:57.270 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.log_readahead_size: 0 2024-04-26T04:49:57.270 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T04:49:57.270 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T04:49:57.270 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T04:49:57.270 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T04:49:57.270 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T04:49:57.270 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T04:49:57.271 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.max_background_jobs: 2 2024-04-26T04:49:57.271 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.max_background_compactions: -1 2024-04-26T04:49:57.271 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.max_subcompactions: 1 2024-04-26T04:49:57.271 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T04:49:57.271 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T04:49:57.271 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T04:49:57.271 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T04:49:57.271 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T04:49:57.271 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T04:49:57.271 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T04:49:57.271 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T04:49:57.271 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.max_open_files: -1 2024-04-26T04:49:57.271 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T04:49:57.271 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T04:49:57.272 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T04:49:57.272 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T04:49:57.272 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.max_background_flushes: -1 2024-04-26T04:49:57.272 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Compression algorithms supported: 2024-04-26T04:49:57.272 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T04:49:57.272 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: kZSTD supported: 0 2024-04-26T04:49:57.272 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: kXpressCompression supported: 0 2024-04-26T04:49:57.272 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T04:49:57.272 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: kZlibCompression supported: 1 2024-04-26T04:49:57.272 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: kSnappyCompression supported: 1 2024-04-26T04:49:57.272 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: kLZ4Compression supported: 1 2024-04-26T04:49:57.272 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: kBZip2Compression supported: 0 2024-04-26T04:49:57.272 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T04:49:57.272 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2024-04-26T04:49:57.273 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-04-26T04:49:57.273 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T04:49:57.273 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.merge_operator: 2024-04-26T04:49:57.273 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.compaction_filter: None 2024-04-26T04:49:57.273 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T04:49:57.273 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T04:49:57.273 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T04:49:57.273 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T04:49:57.273 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5560bcbbcf58) 2024-04-26T04:49:57.273 INFO:journalctl@ceph.mon.c.smithi012.stdout: cache_index_and_filter_blocks: 1 2024-04-26T04:49:57.273 INFO:journalctl@ceph.mon.c.smithi012.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T04:49:57.273 INFO:journalctl@ceph.mon.c.smithi012.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T04:49:57.273 INFO:journalctl@ceph.mon.c.smithi012.stdout: pin_top_level_index_and_filter: 1 2024-04-26T04:49:57.273 INFO:journalctl@ceph.mon.c.smithi012.stdout: index_type: 0 2024-04-26T04:49:57.273 INFO:journalctl@ceph.mon.c.smithi012.stdout: data_block_index_type: 0 2024-04-26T04:49:57.274 INFO:journalctl@ceph.mon.c.smithi012.stdout: index_shortening: 1 2024-04-26T04:49:57.274 INFO:journalctl@ceph.mon.c.smithi012.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T04:49:57.274 INFO:journalctl@ceph.mon.c.smithi012.stdout: hash_index_allow_collision: 1 2024-04-26T04:49:57.274 INFO:journalctl@ceph.mon.c.smithi012.stdout: checksum: 1 2024-04-26T04:49:57.274 INFO:journalctl@ceph.mon.c.smithi012.stdout: no_block_cache: 0 2024-04-26T04:49:57.274 INFO:journalctl@ceph.mon.c.smithi012.stdout: block_cache: 0x5560bcbef090 2024-04-26T04:49:57.274 INFO:journalctl@ceph.mon.c.smithi012.stdout: block_cache_name: BinnedLRUCache 2024-04-26T04:49:57.274 INFO:journalctl@ceph.mon.c.smithi012.stdout: block_cache_options: 2024-04-26T04:49:57.274 INFO:journalctl@ceph.mon.c.smithi012.stdout: capacity : 536870912 2024-04-26T04:49:57.274 INFO:journalctl@ceph.mon.c.smithi012.stdout: num_shard_bits : 4 2024-04-26T04:49:57.274 INFO:journalctl@ceph.mon.c.smithi012.stdout: strict_capacity_limit : 0 2024-04-26T04:49:57.274 INFO:journalctl@ceph.mon.c.smithi012.stdout: high_pri_pool_ratio: 0.000 2024-04-26T04:49:57.274 INFO:journalctl@ceph.mon.c.smithi012.stdout: block_cache_compressed: (nil) 2024-04-26T04:49:57.274 INFO:journalctl@ceph.mon.c.smithi012.stdout: persistent_cache: (nil) 2024-04-26T04:49:57.275 INFO:journalctl@ceph.mon.c.smithi012.stdout: block_size: 4096 2024-04-26T04:49:57.275 INFO:journalctl@ceph.mon.c.smithi012.stdout: block_size_deviation: 10 2024-04-26T04:49:57.275 INFO:journalctl@ceph.mon.c.smithi012.stdout: block_restart_interval: 16 2024-04-26T04:49:57.275 INFO:journalctl@ceph.mon.c.smithi012.stdout: index_block_restart_interval: 1 2024-04-26T04:49:57.275 INFO:journalctl@ceph.mon.c.smithi012.stdout: metadata_block_size: 4096 2024-04-26T04:49:57.275 INFO:journalctl@ceph.mon.c.smithi012.stdout: partition_filters: 0 2024-04-26T04:49:57.275 INFO:journalctl@ceph.mon.c.smithi012.stdout: use_delta_encoding: 1 2024-04-26T04:49:57.275 INFO:journalctl@ceph.mon.c.smithi012.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-26T04:49:57.275 INFO:journalctl@ceph.mon.c.smithi012.stdout: whole_key_filtering: 1 2024-04-26T04:49:57.275 INFO:journalctl@ceph.mon.c.smithi012.stdout: verify_compression: 0 2024-04-26T04:49:57.275 INFO:journalctl@ceph.mon.c.smithi012.stdout: read_amp_bytes_per_bit: 0 2024-04-26T04:49:57.275 INFO:journalctl@ceph.mon.c.smithi012.stdout: format_version: 4 2024-04-26T04:49:57.275 INFO:journalctl@ceph.mon.c.smithi012.stdout: enable_index_compression: 1 2024-04-26T04:49:57.275 INFO:journalctl@ceph.mon.c.smithi012.stdout: block_align: 0 2024-04-26T04:49:57.275 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T04:49:57.276 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T04:49:57.276 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.compression: NoCompression 2024-04-26T04:49:57.276 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T04:49:57.276 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T04:49:57.276 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T04:49:57.276 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.num_levels: 7 2024-04-26T04:49:57.276 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T04:49:57.276 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T04:49:57.276 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T04:49:57.276 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T04:49:57.276 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T04:49:57.276 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T04:49:57.276 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T04:49:57.276 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:49:57.277 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T04:49:57.277 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T04:49:57.277 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T04:49:57.277 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T04:49:57.277 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T04:49:57.277 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T04:49:57.277 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:49:57.277 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T04:49:57.277 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T04:49:57.277 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T04:49:57.277 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T04:49:57.277 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T04:49:57.277 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T04:49:57.277 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T04:49:57.278 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T04:49:57.278 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T04:49:57.278 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T04:49:57.278 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T04:49:57.278 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T04:49:57.278 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T04:49:57.278 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T04:49:57.278 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T04:49:57.278 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T04:49:57.278 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T04:49:57.278 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T04:49:57.278 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T04:49:57.278 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.arena_block_size: 4194304 2024-04-26T04:49:57.278 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T04:49:57.279 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T04:49:57.279 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-26T04:49:57.279 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T04:49:57.279 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T04:49:57.279 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T04:49:57.279 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T04:49:57.279 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T04:49:57.279 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T04:49:57.279 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T04:49:57.279 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T04:49:57.279 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T04:49:57.279 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T04:49:57.279 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T04:49:57.279 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.table_properties_collectors: 2024-04-26T04:49:57.279 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.inplace_update_support: 0 2024-04-26T04:49:57.280 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T04:49:57.280 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T04:49:57.280 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T04:49:57.280 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T04:49:57.280 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.bloom_locality: 0 2024-04-26T04:49:57.280 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.max_successive_merges: 0 2024-04-26T04:49:57.280 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T04:49:57.280 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T04:49:57.280 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T04:49:57.280 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T04:49:57.280 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.ttl: 2592000 2024-04-26T04:49:57.280 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T04:49:57.280 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.enable_blob_files: false 2024-04-26T04:49:57.280 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.min_blob_size: 0 2024-04-26T04:49:57.281 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T04:49:57.281 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T04:49:57.281 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T04:49:57.281 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T04:49:57.281 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T04:49:57.281 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-04-26T04:49:57.281 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-04-26T04:49:57.281 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106996930955, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-04-26T04:49:57.281 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-04-26T04:49:57.281 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-26T04:49:57.281 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106996931260, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1714106996, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "5f895fa6-cde5-4de8-91cc-33f05743dd6b", "db_session_id": "GU20K0DWKZGU0ZQQ5U29"}} 2024-04-26T04:49:57.281 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-04-26T04:49:57.282 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106996931603, "job": 1, "event": "recovery_finished"} 2024-04-26T04:49:57.282 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T04:49:57.282 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x5560bcb83180 2024-04-26T04:49:57.282 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: DB pointer 0x5560bcc58000 2024-04-26T04:49:57.282 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-04-26T04:49:57.282 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-26T04:49:57.282 INFO:journalctl@ceph.mon.c.smithi012.stdout: ** DB Stats ** 2024-04-26T04:49:57.282 INFO:journalctl@ceph.mon.c.smithi012.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:49:57.282 INFO:journalctl@ceph.mon.c.smithi012.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T04:49:57.282 INFO:journalctl@ceph.mon.c.smithi012.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T04:49:57.282 INFO:journalctl@ceph.mon.c.smithi012.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:49:57.282 INFO:journalctl@ceph.mon.c.smithi012.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T04:49:57.282 INFO:journalctl@ceph.mon.c.smithi012.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-26T04:49:57.283 INFO:journalctl@ceph.mon.c.smithi012.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:49:57.283 INFO:journalctl@ceph.mon.c.smithi012.stdout: 2024-04-26T04:49:57.283 INFO:journalctl@ceph.mon.c.smithi012.stdout: ** Compaction Stats [default] ** 2024-04-26T04:49:57.283 INFO:journalctl@ceph.mon.c.smithi012.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:49:57.283 INFO:journalctl@ceph.mon.c.smithi012.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:49:57.283 INFO:journalctl@ceph.mon.c.smithi012.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-04-26T04:49:57.283 INFO:journalctl@ceph.mon.c.smithi012.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-04-26T04:49:57.283 INFO:journalctl@ceph.mon.c.smithi012.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-04-26T04:49:57.283 INFO:journalctl@ceph.mon.c.smithi012.stdout: 2024-04-26T04:49:57.283 INFO:journalctl@ceph.mon.c.smithi012.stdout: ** Compaction Stats [default] ** 2024-04-26T04:49:57.283 INFO:journalctl@ceph.mon.c.smithi012.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:49:57.283 INFO:journalctl@ceph.mon.c.smithi012.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:49:57.283 INFO:journalctl@ceph.mon.c.smithi012.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-04-26T04:49:57.283 INFO:journalctl@ceph.mon.c.smithi012.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:49:57.283 INFO:journalctl@ceph.mon.c.smithi012.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-26T04:49:57.284 INFO:journalctl@ceph.mon.c.smithi012.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T04:49:57.284 INFO:journalctl@ceph.mon.c.smithi012.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T04:49:57.284 INFO:journalctl@ceph.mon.c.smithi012.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T04:49:57.284 INFO:journalctl@ceph.mon.c.smithi012.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T04:49:57.284 INFO:journalctl@ceph.mon.c.smithi012.stdout: Cumulative compaction: 0.00 GB write, 0.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:49:57.284 INFO:journalctl@ceph.mon.c.smithi012.stdout: Interval compaction: 0.00 GB write, 0.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:49:57.284 INFO:journalctl@ceph.mon.c.smithi012.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T04:49:57.284 INFO:journalctl@ceph.mon.c.smithi012.stdout: 2024-04-26T04:49:57.284 INFO:journalctl@ceph.mon.c.smithi012.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T04:49:57.284 INFO:journalctl@ceph.mon.c.smithi012.stdout: 2024-04-26T04:49:57.284 INFO:journalctl@ceph.mon.c.smithi012.stdout: ** Compaction Stats [default] ** 2024-04-26T04:49:57.284 INFO:journalctl@ceph.mon.c.smithi012.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:49:57.284 INFO:journalctl@ceph.mon.c.smithi012.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:49:57.284 INFO:journalctl@ceph.mon.c.smithi012.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-04-26T04:49:57.285 INFO:journalctl@ceph.mon.c.smithi012.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-04-26T04:49:57.285 INFO:journalctl@ceph.mon.c.smithi012.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-26T04:49:57.285 INFO:journalctl@ceph.mon.c.smithi012.stdout: 2024-04-26T04:49:57.285 INFO:journalctl@ceph.mon.c.smithi012.stdout: ** Compaction Stats [default] ** 2024-04-26T04:49:57.285 INFO:journalctl@ceph.mon.c.smithi012.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:49:57.285 INFO:journalctl@ceph.mon.c.smithi012.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:49:57.285 INFO:journalctl@ceph.mon.c.smithi012.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-04-26T04:49:57.285 INFO:journalctl@ceph.mon.c.smithi012.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:49:57.285 INFO:journalctl@ceph.mon.c.smithi012.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-26T04:49:57.285 INFO:journalctl@ceph.mon.c.smithi012.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T04:49:57.285 INFO:journalctl@ceph.mon.c.smithi012.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T04:49:57.285 INFO:journalctl@ceph.mon.c.smithi012.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T04:49:57.285 INFO:journalctl@ceph.mon.c.smithi012.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T04:49:57.285 INFO:journalctl@ceph.mon.c.smithi012.stdout: Cumulative compaction: 0.00 GB write, 0.61 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:49:57.286 INFO:journalctl@ceph.mon.c.smithi012.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:49:57.286 INFO:journalctl@ceph.mon.c.smithi012.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T04:49:57.286 INFO:journalctl@ceph.mon.c.smithi012.stdout: 2024-04-26T04:49:57.286 INFO:journalctl@ceph.mon.c.smithi012.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T04:49:57.286 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-04-26T04:49:57.286 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: using public_addrv [v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0] 2024-04-26T04:49:57.286 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: starting mon.c rank -1 at public addrs [v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0] at bind addrs [v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T04:49:57.286 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: mon.c@-1(???) e0 preinit fsid 0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T04:49:57.286 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: mon.c@-1(synchronizing).mds e1 new map 2024-04-26T04:49:57.286 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: mon.c@-1(synchronizing).mds e1 print_map 2024-04-26T04:49:57.286 INFO:journalctl@ceph.mon.c.smithi012.stdout: e1 2024-04-26T04:49:57.286 INFO:journalctl@ceph.mon.c.smithi012.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T04:49:57.286 INFO:journalctl@ceph.mon.c.smithi012.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T04:49:57.286 INFO:journalctl@ceph.mon.c.smithi012.stdout: legacy client fscid: -1 2024-04-26T04:49:57.287 INFO:journalctl@ceph.mon.c.smithi012.stdout: 2024-04-26T04:49:57.287 INFO:journalctl@ceph.mon.c.smithi012.stdout: No filesystems configured 2024-04-26T04:49:57.287 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-04-26T04:49:57.287 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-04-26T04:49:57.287 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-04-26T04:49:57.287 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-04-26T04:49:57.287 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-04-26T04:49:57.287 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-04-26T04:49:57.287 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-26T04:49:57.287 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:49:57.287 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:49:57.287 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:49:57.287 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: Updating smithi116:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:49:57.287 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: Updating smithi116:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:49:57.287 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/2612484146' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-26T04:49:57.288 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:49:57.288 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:49:57.288 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi012:172.21.15.12=a;smithi012:[v2:172.21.15.12:3301,v1:172.21.15.12:6790]=c;smithi116:172.21.15.116=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:49:57.288 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: Saving service mon spec with placement smithi012:172.21.15.12=a;smithi012:[v2:172.21.15.12:3301,v1:172.21.15.12:6790]=c;smithi116:172.21.15.116=b;count:3 2024-04-26T04:49:57.288 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:57.288 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:57.288 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:57.288 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:49:57.288 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:57.288 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:49:57.288 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:57.288 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: Deploying daemon mon.b on smithi116 2024-04-26T04:49:57.288 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:49:57.288 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:49:57.289 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:49:57.289 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:49:57.289 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: mon.a calling monitor election 2024-04-26T04:49:57.289 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:49:57.289 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:49:57.289 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:49:57.289 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:49:57.289 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: mon.b calling monitor election 2024-04-26T04:49:57.289 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:49:57.289 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:49:57.289 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:49:57.289 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:49:57.289 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-26T04:49:57.289 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: monmap e2: 2 mons at {a=[v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0],b=[v2:172.21.15.116:3300/0,v1:172.21.15.116:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:49:57.290 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: fsmap 2024-04-26T04:49:57.290 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:49:57.290 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: mgrmap e13: y(active, since 37s) 2024-04-26T04:49:57.290 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: overall HEALTH_OK 2024-04-26T04:49:57.290 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:57.290 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:57.290 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:49:57.290 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:49:57.290 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:57.290 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: Deploying daemon mon.c on smithi012 2024-04-26T04:49:57.290 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:49:57.290 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:49:57.290 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: from='client.? 172.21.15.116:0/3214300913' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T04:49:57.290 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 ceph-mon[37567]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-04-26T04:49:57.291 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 bash[37548]: 0f03d4c16528513e9afcd9c7cf63732f1f25d5bb1bcb3ff09079b69b6a66d946 2024-04-26T04:49:57.291 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:49:56 smithi012 systemd[1]: Started Ceph mon.c for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:49:57.335 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:50:02.481 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:02 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:50:02.481 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:02 smithi116 ceph-mon[29897]: mon.a calling monitor election 2024-04-26T04:50:02.481 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:02 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:50:02.481 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:02 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:02.482 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:02 smithi116 ceph-mon[29897]: mon.b calling monitor election 2024-04-26T04:50:02.482 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:02 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:02.482 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:02 smithi116 ceph-mon[29897]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:02.482 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:02 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:02.482 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:02 smithi116 ceph-mon[29897]: mon.c calling monitor election 2024-04-26T04:50:02.482 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:02 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:02.482 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:02 smithi116 ceph-mon[29897]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:02.482 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:02 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:02.483 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:02 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:02.483 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:02 smithi116 ceph-mon[29897]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:50:02.483 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:02 smithi116 ceph-mon[29897]: monmap e3: 3 mons at {a=[v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0],b=[v2:172.21.15.116:3300/0,v1:172.21.15.116:6789/0],c=[v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:50:02.483 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:02 smithi116 ceph-mon[29897]: fsmap 2024-04-26T04:50:02.483 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:02 smithi116 ceph-mon[29897]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:50:02.483 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:02 smithi116 ceph-mon[29897]: mgrmap e13: y(active, since 45s) 2024-04-26T04:50:02.483 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:02 smithi116 ceph-mon[29897]: overall HEALTH_OK 2024-04-26T04:50:02.483 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:02 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:02.483 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:02 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:02.499 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:50:02.499 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[37567]: mon.a calling monitor election 2024-04-26T04:50:02.499 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:50:02.499 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:02.500 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[37567]: mon.b calling monitor election 2024-04-26T04:50:02.500 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:02.500 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[37567]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:02.500 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:02.500 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[37567]: mon.c calling monitor election 2024-04-26T04:50:02.500 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:02.500 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[37567]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:02.500 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:02.500 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:02.500 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[37567]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:50:02.500 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[37567]: monmap e3: 3 mons at {a=[v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0],b=[v2:172.21.15.116:3300/0,v1:172.21.15.116:6789/0],c=[v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:50:02.501 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[37567]: fsmap 2024-04-26T04:50:02.501 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[37567]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:50:02.501 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[37567]: mgrmap e13: y(active, since 45s) 2024-04-26T04:50:02.501 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[37567]: overall HEALTH_OK 2024-04-26T04:50:02.501 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:02.501 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:02.501 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:50:02.501 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[26617]: mon.a calling monitor election 2024-04-26T04:50:02.501 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:50:02.501 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:02.501 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[26617]: mon.b calling monitor election 2024-04-26T04:50:02.502 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:02.502 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[26617]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:02.502 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:02.502 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[26617]: mon.c calling monitor election 2024-04-26T04:50:02.502 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:02.502 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[26617]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:02.502 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:02.502 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:02.502 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[26617]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:50:02.502 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[26617]: monmap e3: 3 mons at {a=[v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0],b=[v2:172.21.15.116:3300/0,v1:172.21.15.116:6789/0],c=[v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:50:02.502 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[26617]: fsmap 2024-04-26T04:50:02.502 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[26617]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:50:02.502 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[26617]: mgrmap e13: y(active, since 45s) 2024-04-26T04:50:02.503 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[26617]: overall HEALTH_OK 2024-04-26T04:50:02.503 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:02.503 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:02 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:02.577 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-26T04:50:02.578 INFO:teuthology.orchestra.run.smithi116.stdout:{"epoch":3,"fsid":"0b9350c6-0388-11ef-bc93-c7b262605968","modified":"2024-04-26T04:49:56.953431Z","created":"2024-04-26T04:48:15.190556Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:3300","nonce":0},{"type":"v1","addr":"172.21.15.12:6789","nonce":0}]},"addr":"172.21.15.12:6789/0","public_addr":"172.21.15.12:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:3300","nonce":0},{"type":"v1","addr":"172.21.15.116:6789","nonce":0}]},"addr":"172.21.15.116:6789/0","public_addr":"172.21.15.116:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:3301","nonce":0},{"type":"v1","addr":"172.21.15.12:6790","nonce":0}]},"addr":"172.21.15.12:6790/0","public_addr":"172.21.15.12:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-04-26T04:50:02.580 INFO:teuthology.orchestra.run.smithi116.stderr:dumped monmap epoch 3 2024-04-26T04:50:03.053 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:03 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:03.054 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:03 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:03.054 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:03 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:03.054 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:03 smithi012 ceph-mon[26617]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:03.054 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:03 smithi012 ceph-mon[26617]: from='client.? 172.21.15.116:0/2468284404' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T04:50:03.054 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:03 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:03.054 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:03 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:03.054 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:03 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:03.055 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:03 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:03.055 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:03 smithi012 ceph-mon[37567]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:03.055 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:03 smithi012 ceph-mon[37567]: from='client.? 172.21.15.116:0/2468284404' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T04:50:03.055 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:03 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:03.291 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:03 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:03.291 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:03 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:03.292 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:03 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:03.292 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:03 smithi116 ceph-mon[29897]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:03.292 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:03 smithi116 ceph-mon[29897]: from='client.? 172.21.15.116:0/2468284404' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T04:50:03.292 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:03 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:50:03.293 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-04-26T04:50:03.293 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph config generate-minimal-conf 2024-04-26T04:50:03.557 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:50:04.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:04 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:04.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:04 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:04.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:04 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:04.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:04 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:04.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:04 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:04.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:04 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:04.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:04 smithi116 ceph-mon[29897]: Updating smithi012:/etc/ceph/ceph.conf 2024-04-26T04:50:04.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:04 smithi116 ceph-mon[29897]: Updating smithi116:/etc/ceph/ceph.conf 2024-04-26T04:50:04.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:04 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:04.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:04 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:04.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:04 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:04.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:04 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:04.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:04 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:04.743 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:04 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:50:04.743 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:04 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:50:04.743 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:04 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:04.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:04 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:04.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:04 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:04.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:04 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:04.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:04 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:04.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:04 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:04.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:04 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:04.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:04 smithi012 ceph-mon[26617]: Updating smithi012:/etc/ceph/ceph.conf 2024-04-26T04:50:04.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:04 smithi012 ceph-mon[26617]: Updating smithi116:/etc/ceph/ceph.conf 2024-04-26T04:50:04.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:04 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:04.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:04 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:04.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:04 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:04.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:04 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:04.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:04 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:04.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:04 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:50:04.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:04 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:50:04.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:04 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:04.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:04 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:04.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:04 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:04.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:04 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:04.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:04 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:04.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:04 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:04.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:04 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:04.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:04 smithi012 ceph-mon[37567]: Updating smithi012:/etc/ceph/ceph.conf 2024-04-26T04:50:04.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:04 smithi012 ceph-mon[37567]: Updating smithi116:/etc/ceph/ceph.conf 2024-04-26T04:50:04.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:04 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:04.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:04 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:04.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:04 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:04.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:04 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:04.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:04 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:04.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:04 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:50:04.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:04 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:50:04.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:04 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:05.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:05 smithi116 ceph-mon[29897]: Updating smithi116:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:50:05.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:05 smithi116 ceph-mon[29897]: Updating smithi012:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:50:05.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:05 smithi116 ceph-mon[29897]: Reconfiguring mon.a (unknown last config time)... 2024-04-26T04:50:05.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:05 smithi116 ceph-mon[29897]: Reconfiguring daemon mon.a on smithi012 2024-04-26T04:50:05.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:05 smithi116 ceph-mon[29897]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:05.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:05 smithi012 ceph-mon[26617]: Updating smithi116:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:50:05.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:05 smithi012 ceph-mon[26617]: Updating smithi012:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:50:05.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:05 smithi012 ceph-mon[26617]: Reconfiguring mon.a (unknown last config time)... 2024-04-26T04:50:05.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:05 smithi012 ceph-mon[26617]: Reconfiguring daemon mon.a on smithi012 2024-04-26T04:50:05.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:05 smithi012 ceph-mon[26617]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:05.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:05 smithi012 ceph-mon[37567]: Updating smithi116:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:50:05.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:05 smithi012 ceph-mon[37567]: Updating smithi012:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:50:05.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:05 smithi012 ceph-mon[37567]: Reconfiguring mon.a (unknown last config time)... 2024-04-26T04:50:05.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:05 smithi012 ceph-mon[37567]: Reconfiguring daemon mon.a on smithi012 2024-04-26T04:50:05.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:05 smithi012 ceph-mon[37567]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:06.038 INFO:teuthology.orchestra.run.smithi012.stdout:# minimal ceph.conf for 0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T04:50:06.039 INFO:teuthology.orchestra.run.smithi012.stdout:[global] 2024-04-26T04:50:06.039 INFO:teuthology.orchestra.run.smithi012.stdout: fsid = 0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T04:50:06.039 INFO:teuthology.orchestra.run.smithi012.stdout: mon_host = [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] [v2:172.21.15.116:3300/0,v1:172.21.15.116:6789/0] [v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0] 2024-04-26T04:50:06.568 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:06 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/636636492' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:06.568 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:06 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:06.568 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:06 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:06.568 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:06 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:50:06.568 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:06 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:50:06.568 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:06 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:06.568 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:06 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/636636492' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:06.568 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:06 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:06.569 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:06 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:06.569 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:06 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:50:06.569 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:06 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:50:06.569 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:06 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:06.595 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-04-26T04:50:06.596 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-04-26T04:50:06.596 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/etc/ceph/ceph.conf 2024-04-26T04:50:06.623 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-04-26T04:50:06.623 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:50:06.690 DEBUG:teuthology.orchestra.run.smithi116:> set -ex 2024-04-26T04:50:06.690 DEBUG:teuthology.orchestra.run.smithi116:> sudo dd of=/etc/ceph/ceph.conf 2024-04-26T04:50:06.716 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:06 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/636636492' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:06.716 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:06 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:06.716 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:06 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:06.716 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:06 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:50:06.716 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:06 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:50:06.716 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:06 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:06.721 DEBUG:teuthology.orchestra.run.smithi116:> set -ex 2024-04-26T04:50:06.721 DEBUG:teuthology.orchestra.run.smithi116:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:50:06.788 INFO:tasks.cephadm:Adding mgr.y on smithi012 2024-04-26T04:50:06.788 INFO:tasks.cephadm:Adding mgr.x on smithi116 2024-04-26T04:50:06.788 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph orch apply mgr '2;smithi012=y;smithi116=x' 2024-04-26T04:50:07.076 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:50:07.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:07 smithi116 ceph-mon[29897]: Reconfiguring mon.c (monmap changed)... 2024-04-26T04:50:07.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:07 smithi116 ceph-mon[29897]: Reconfiguring daemon mon.c on smithi012 2024-04-26T04:50:07.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:07 smithi116 ceph-mon[29897]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:07.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:07 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:08.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:07 smithi012 ceph-mon[26617]: Reconfiguring mon.c (monmap changed)... 2024-04-26T04:50:08.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:07 smithi012 ceph-mon[26617]: Reconfiguring daemon mon.c on smithi012 2024-04-26T04:50:08.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:07 smithi012 ceph-mon[26617]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:08.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:07 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:08.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:07 smithi012 ceph-mon[37567]: Reconfiguring mon.c (monmap changed)... 2024-04-26T04:50:08.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:07 smithi012 ceph-mon[37567]: Reconfiguring daemon mon.c on smithi012 2024-04-26T04:50:08.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:07 smithi012 ceph-mon[37567]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:08.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:07 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:08.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:08 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:08.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:08 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:08.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:08 smithi116 ceph-mon[29897]: Reconfiguring mon.b (monmap changed)... 2024-04-26T04:50:08.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:08 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:50:08.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:08 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:50:08.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:08 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:08.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:08 smithi116 ceph-mon[29897]: Reconfiguring daemon mon.b on smithi116 2024-04-26T04:50:08.992 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:08 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:08.992 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:08 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:08.992 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:08 smithi012 ceph-mon[37567]: Reconfiguring mon.b (monmap changed)... 2024-04-26T04:50:08.993 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:08 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:50:08.993 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:08 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:50:08.993 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:08 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:08.993 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:08 smithi012 ceph-mon[37567]: Reconfiguring daemon mon.b on smithi116 2024-04-26T04:50:08.993 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:08 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:08.993 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:08 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:08.993 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:08 smithi012 ceph-mon[26617]: Reconfiguring mon.b (monmap changed)... 2024-04-26T04:50:08.993 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:08 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:50:08.993 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:08 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:50:08.993 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:08 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:08.993 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:08 smithi012 ceph-mon[26617]: Reconfiguring daemon mon.b on smithi116 2024-04-26T04:50:09.842 INFO:teuthology.orchestra.run.smithi116.stdout:Scheduled mgr update... 2024-04-26T04:50:09.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:09 smithi116 ceph-mon[29897]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:09.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:09 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:09.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:09 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:09.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:09 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:09.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:09 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:09.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:09 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:09.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:09 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:10.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:09 smithi012 ceph-mon[26617]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:10.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:09 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:10.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:09 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:10.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:09 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:10.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:09 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:10.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:09 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:10.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:09 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:10.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:09 smithi012 ceph-mon[37567]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:10.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:09 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:10.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:09 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:10.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:09 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:10.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:09 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:10.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:09 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:10.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:09 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:10.395 DEBUG:teuthology.orchestra.run.smithi116:mgr.x> sudo journalctl -f -n 0 -u ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mgr.x.service 2024-04-26T04:50:10.397 INFO:tasks.cephadm:Deploying OSDs... 2024-04-26T04:50:10.397 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-04-26T04:50:10.397 DEBUG:teuthology.orchestra.run.smithi012:> dd if=/scratch_devs of=/dev/stdout 2024-04-26T04:50:10.415 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-26T04:50:10.415 DEBUG:teuthology.orchestra.run.smithi012:> stat /dev/vg_nvme/lv_1 2024-04-26T04:50:10.471 INFO:teuthology.orchestra.run.smithi012.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-26T04:50:10.472 INFO:teuthology.orchestra.run.smithi012.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:50:10.472 INFO:teuthology.orchestra.run.smithi012.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-04-26T04:50:10.472 INFO:teuthology.orchestra.run.smithi012.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:50:10.472 INFO:teuthology.orchestra.run.smithi012.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:50:10.472 INFO:teuthology.orchestra.run.smithi012.stdout:Access: 2024-04-26 04:49:56.391628512 +0000 2024-04-26T04:50:10.472 INFO:teuthology.orchestra.run.smithi012.stdout:Modify: 2024-04-26 04:44:45.435776455 +0000 2024-04-26T04:50:10.472 INFO:teuthology.orchestra.run.smithi012.stdout:Change: 2024-04-26 04:44:45.435776455 +0000 2024-04-26T04:50:10.472 INFO:teuthology.orchestra.run.smithi012.stdout: Birth: 2024-04-26 04:44:45.435776455 +0000 2024-04-26T04:50:10.472 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-26T04:50:10.537 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records in 2024-04-26T04:50:10.538 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records out 2024-04-26T04:50:10.538 INFO:teuthology.orchestra.run.smithi012.stderr:512 bytes copied, 0.000272708 s, 1.9 MB/s 2024-04-26T04:50:10.539 DEBUG:teuthology.orchestra.run.smithi012:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-26T04:50:10.597 DEBUG:teuthology.orchestra.run.smithi012:> stat /dev/vg_nvme/lv_2 2024-04-26T04:50:10.653 INFO:teuthology.orchestra.run.smithi012.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-26T04:50:10.653 INFO:teuthology.orchestra.run.smithi012.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:50:10.653 INFO:teuthology.orchestra.run.smithi012.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-04-26T04:50:10.653 INFO:teuthology.orchestra.run.smithi012.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:50:10.653 INFO:teuthology.orchestra.run.smithi012.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:50:10.653 INFO:teuthology.orchestra.run.smithi012.stdout:Access: 2024-04-26 04:49:56.391628512 +0000 2024-04-26T04:50:10.653 INFO:teuthology.orchestra.run.smithi012.stdout:Modify: 2024-04-26 04:44:45.784773651 +0000 2024-04-26T04:50:10.653 INFO:teuthology.orchestra.run.smithi012.stdout:Change: 2024-04-26 04:44:45.784773651 +0000 2024-04-26T04:50:10.653 INFO:teuthology.orchestra.run.smithi012.stdout: Birth: 2024-04-26 04:44:45.784773651 +0000 2024-04-26T04:50:10.654 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-26T04:50:10.716 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records in 2024-04-26T04:50:10.717 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records out 2024-04-26T04:50:10.717 INFO:teuthology.orchestra.run.smithi012.stderr:512 bytes copied, 0.000182577 s, 2.8 MB/s 2024-04-26T04:50:10.718 DEBUG:teuthology.orchestra.run.smithi012:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-26T04:50:10.774 DEBUG:teuthology.orchestra.run.smithi012:> stat /dev/vg_nvme/lv_3 2024-04-26T04:50:10.829 INFO:teuthology.orchestra.run.smithi012.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-26T04:50:10.829 INFO:teuthology.orchestra.run.smithi012.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:50:10.829 INFO:teuthology.orchestra.run.smithi012.stdout:Device: 5h/5d Inode: 704 Links: 1 2024-04-26T04:50:10.829 INFO:teuthology.orchestra.run.smithi012.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:50:10.829 INFO:teuthology.orchestra.run.smithi012.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:50:10.829 INFO:teuthology.orchestra.run.smithi012.stdout:Access: 2024-04-26 04:49:56.391628512 +0000 2024-04-26T04:50:10.829 INFO:teuthology.orchestra.run.smithi012.stdout:Modify: 2024-04-26 04:44:46.138770806 +0000 2024-04-26T04:50:10.829 INFO:teuthology.orchestra.run.smithi012.stdout:Change: 2024-04-26 04:44:46.138770806 +0000 2024-04-26T04:50:10.829 INFO:teuthology.orchestra.run.smithi012.stdout: Birth: 2024-04-26 04:44:46.138770806 +0000 2024-04-26T04:50:10.829 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-26T04:50:10.892 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records in 2024-04-26T04:50:10.893 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records out 2024-04-26T04:50:10.893 INFO:teuthology.orchestra.run.smithi012.stderr:512 bytes copied, 0.00019397 s, 2.6 MB/s 2024-04-26T04:50:10.894 DEBUG:teuthology.orchestra.run.smithi012:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-26T04:50:10.924 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:10 smithi116 ceph-mon[29897]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi012=y;smithi116=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:50:10.925 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:10 smithi116 ceph-mon[29897]: Saving service mgr spec with placement smithi012=y;smithi116=x;count:2 2024-04-26T04:50:10.925 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:10 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:10.925 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:10 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:10.925 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:10 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:10.925 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:10 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:10.925 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:10 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:10.925 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:10 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:50:10.925 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:10 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-26T04:50:10.925 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:10 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:50:10.925 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:10 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:10.925 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:10 smithi116 ceph-mon[29897]: Deploying daemon mgr.x on smithi116 2024-04-26T04:50:10.925 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:10 smithi116 ceph-mon[29897]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:10.950 DEBUG:teuthology.orchestra.run.smithi012:> stat /dev/vg_nvme/lv_4 2024-04-26T04:50:11.006 INFO:teuthology.orchestra.run.smithi012.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-26T04:50:11.006 INFO:teuthology.orchestra.run.smithi012.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:50:11.006 INFO:teuthology.orchestra.run.smithi012.stdout:Device: 5h/5d Inode: 721 Links: 1 2024-04-26T04:50:11.006 INFO:teuthology.orchestra.run.smithi012.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:50:11.006 INFO:teuthology.orchestra.run.smithi012.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:50:11.006 INFO:teuthology.orchestra.run.smithi012.stdout:Access: 2024-04-26 04:49:56.392628504 +0000 2024-04-26T04:50:11.006 INFO:teuthology.orchestra.run.smithi012.stdout:Modify: 2024-04-26 04:44:46.492767961 +0000 2024-04-26T04:50:11.006 INFO:teuthology.orchestra.run.smithi012.stdout:Change: 2024-04-26 04:44:46.492767961 +0000 2024-04-26T04:50:11.006 INFO:teuthology.orchestra.run.smithi012.stdout: Birth: 2024-04-26 04:44:46.492767961 +0000 2024-04-26T04:50:11.007 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-26T04:50:11.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:10 smithi012 ceph-mon[26617]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi012=y;smithi116=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:50:11.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:10 smithi012 ceph-mon[26617]: Saving service mgr spec with placement smithi012=y;smithi116=x;count:2 2024-04-26T04:50:11.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:10 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:11.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:10 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:11.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:10 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:11.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:10 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:11.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:10 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:11.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:10 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:50:11.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:10 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-26T04:50:11.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:10 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:50:11.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:10 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:11.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:10 smithi012 ceph-mon[26617]: Deploying daemon mgr.x on smithi116 2024-04-26T04:50:11.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:10 smithi012 ceph-mon[26617]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:11.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:10 smithi012 ceph-mon[37567]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi012=y;smithi116=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:50:11.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:10 smithi012 ceph-mon[37567]: Saving service mgr spec with placement smithi012=y;smithi116=x;count:2 2024-04-26T04:50:11.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:10 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:11.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:10 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:11.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:10 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:11.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:10 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:11.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:10 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:11.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:10 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:50:11.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:10 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-26T04:50:11.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:10 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:50:11.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:10 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:11.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:10 smithi012 ceph-mon[37567]: Deploying daemon mgr.x on smithi116 2024-04-26T04:50:11.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:10 smithi012 ceph-mon[37567]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:11.036 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records in 2024-04-26T04:50:11.036 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records out 2024-04-26T04:50:11.037 INFO:teuthology.orchestra.run.smithi012.stderr:512 bytes copied, 0.000245859 s, 2.1 MB/s 2024-04-26T04:50:11.038 DEBUG:teuthology.orchestra.run.smithi012:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-26T04:50:11.093 DEBUG:teuthology.orchestra.run.smithi116:> set -ex 2024-04-26T04:50:11.093 DEBUG:teuthology.orchestra.run.smithi116:> dd if=/scratch_devs of=/dev/stdout 2024-04-26T04:50:11.109 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-26T04:50:11.109 DEBUG:teuthology.orchestra.run.smithi116:> stat /dev/vg_nvme/lv_1 2024-04-26T04:50:11.165 INFO:teuthology.orchestra.run.smithi116.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-26T04:50:11.165 INFO:teuthology.orchestra.run.smithi116.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:50:11.166 INFO:teuthology.orchestra.run.smithi116.stdout:Device: 5h/5d Inode: 664 Links: 1 2024-04-26T04:50:11.166 INFO:teuthology.orchestra.run.smithi116.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:50:11.166 INFO:teuthology.orchestra.run.smithi116.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:50:11.166 INFO:teuthology.orchestra.run.smithi116.stdout:Access: 2024-04-26 04:49:47.657527476 +0000 2024-04-26T04:50:11.166 INFO:teuthology.orchestra.run.smithi116.stdout:Modify: 2024-04-26 04:44:40.829173200 +0000 2024-04-26T04:50:11.166 INFO:teuthology.orchestra.run.smithi116.stdout:Change: 2024-04-26 04:44:40.829173200 +0000 2024-04-26T04:50:11.166 INFO:teuthology.orchestra.run.smithi116.stdout: Birth: 2024-04-26 04:44:40.829173200 +0000 2024-04-26T04:50:11.166 DEBUG:teuthology.orchestra.run.smithi116:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-26T04:50:11.229 INFO:teuthology.orchestra.run.smithi116.stderr:1+0 records in 2024-04-26T04:50:11.229 INFO:teuthology.orchestra.run.smithi116.stderr:1+0 records out 2024-04-26T04:50:11.229 INFO:teuthology.orchestra.run.smithi116.stderr:512 bytes copied, 0.000188261 s, 2.7 MB/s 2024-04-26T04:50:11.230 DEBUG:teuthology.orchestra.run.smithi116:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-26T04:50:11.286 DEBUG:teuthology.orchestra.run.smithi116:> stat /dev/vg_nvme/lv_2 2024-04-26T04:50:11.340 INFO:teuthology.orchestra.run.smithi116.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-26T04:50:11.340 INFO:teuthology.orchestra.run.smithi116.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:50:11.340 INFO:teuthology.orchestra.run.smithi116.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-04-26T04:50:11.340 INFO:teuthology.orchestra.run.smithi116.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:50:11.340 INFO:teuthology.orchestra.run.smithi116.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:50:11.341 INFO:teuthology.orchestra.run.smithi116.stdout:Access: 2024-04-26 04:49:47.657527476 +0000 2024-04-26T04:50:11.341 INFO:teuthology.orchestra.run.smithi116.stdout:Modify: 2024-04-26 04:44:41.163167619 +0000 2024-04-26T04:50:11.341 INFO:teuthology.orchestra.run.smithi116.stdout:Change: 2024-04-26 04:44:41.163167619 +0000 2024-04-26T04:50:11.341 INFO:teuthology.orchestra.run.smithi116.stdout: Birth: 2024-04-26 04:44:41.163167619 +0000 2024-04-26T04:50:11.341 DEBUG:teuthology.orchestra.run.smithi116:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-26T04:50:11.402 INFO:teuthology.orchestra.run.smithi116.stderr:1+0 records in 2024-04-26T04:50:11.403 INFO:teuthology.orchestra.run.smithi116.stderr:1+0 records out 2024-04-26T04:50:11.403 INFO:teuthology.orchestra.run.smithi116.stderr:512 bytes copied, 0.000217887 s, 2.3 MB/s 2024-04-26T04:50:11.404 DEBUG:teuthology.orchestra.run.smithi116:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-26T04:50:11.459 DEBUG:teuthology.orchestra.run.smithi116:> stat /dev/vg_nvme/lv_3 2024-04-26T04:50:11.514 INFO:teuthology.orchestra.run.smithi116.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-26T04:50:11.514 INFO:teuthology.orchestra.run.smithi116.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:50:11.515 INFO:teuthology.orchestra.run.smithi116.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-04-26T04:50:11.515 INFO:teuthology.orchestra.run.smithi116.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:50:11.515 INFO:teuthology.orchestra.run.smithi116.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:50:11.515 INFO:teuthology.orchestra.run.smithi116.stdout:Access: 2024-04-26 04:49:47.658527460 +0000 2024-04-26T04:50:11.515 INFO:teuthology.orchestra.run.smithi116.stdout:Modify: 2024-04-26 04:44:41.503161938 +0000 2024-04-26T04:50:11.515 INFO:teuthology.orchestra.run.smithi116.stdout:Change: 2024-04-26 04:44:41.503161938 +0000 2024-04-26T04:50:11.515 INFO:teuthology.orchestra.run.smithi116.stdout: Birth: 2024-04-26 04:44:41.503161938 +0000 2024-04-26T04:50:11.515 DEBUG:teuthology.orchestra.run.smithi116:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-26T04:50:11.576 INFO:teuthology.orchestra.run.smithi116.stderr:1+0 records in 2024-04-26T04:50:11.576 INFO:teuthology.orchestra.run.smithi116.stderr:1+0 records out 2024-04-26T04:50:11.576 INFO:teuthology.orchestra.run.smithi116.stderr:512 bytes copied, 0.000218997 s, 2.3 MB/s 2024-04-26T04:50:11.577 DEBUG:teuthology.orchestra.run.smithi116:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-26T04:50:11.636 DEBUG:teuthology.orchestra.run.smithi116:> stat /dev/vg_nvme/lv_4 2024-04-26T04:50:11.692 INFO:teuthology.orchestra.run.smithi116.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-26T04:50:11.692 INFO:teuthology.orchestra.run.smithi116.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:50:11.692 INFO:teuthology.orchestra.run.smithi116.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-04-26T04:50:11.692 INFO:teuthology.orchestra.run.smithi116.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:50:11.692 INFO:teuthology.orchestra.run.smithi116.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:50:11.692 INFO:teuthology.orchestra.run.smithi116.stdout:Access: 2024-04-26 04:49:47.658527460 +0000 2024-04-26T04:50:11.692 INFO:teuthology.orchestra.run.smithi116.stdout:Modify: 2024-04-26 04:44:41.839156324 +0000 2024-04-26T04:50:11.692 INFO:teuthology.orchestra.run.smithi116.stdout:Change: 2024-04-26 04:44:41.839156324 +0000 2024-04-26T04:50:11.693 INFO:teuthology.orchestra.run.smithi116.stdout: Birth: 2024-04-26 04:44:41.839156324 +0000 2024-04-26T04:50:11.693 DEBUG:teuthology.orchestra.run.smithi116:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-26T04:50:11.757 INFO:teuthology.orchestra.run.smithi116.stderr:1+0 records in 2024-04-26T04:50:11.757 INFO:teuthology.orchestra.run.smithi116.stderr:1+0 records out 2024-04-26T04:50:11.757 INFO:teuthology.orchestra.run.smithi116.stderr:512 bytes copied, 0.000190305 s, 2.7 MB/s 2024-04-26T04:50:11.758 DEBUG:teuthology.orchestra.run.smithi116:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-26T04:50:11.818 INFO:tasks.cephadm:Deploying osd.0 on smithi012 with /dev/vg_nvme/lv_4... 2024-04-26T04:50:11.818 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-26T04:50:12.085 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:50:12.241 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:50:12 smithi116 podman[31794]: 2024-04-26 04:50:12.068721545 +0000 UTC m=+0.017666298 image pull 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17 2024-04-26T04:50:12.513 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:50:12 smithi116 podman[31794]: 2024-04-26 04:50:12.364702695 +0000 UTC m=+0.313647436 container init dd1f1c263e39cebcaccfdd511a0ede6634c6eeda321b0af624487a37898404ac (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=quincy-c61472c, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424) 2024-04-26T04:50:12.513 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:50:12 smithi116 podman[31794]: 2024-04-26 04:50:12.367222793 +0000 UTC m=+0.316167534 container start dd1f1c263e39cebcaccfdd511a0ede6634c6eeda321b0af624487a37898404ac (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=quincy-c61472c, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T04:50:12.513 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:50:12 smithi116 ceph-mgr[31814]: -- 172.21.15.116:0/494839581 <== mon.1 v2:172.21.15.116:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55a5c5bca9c0 con 0x55a5c5ba0c00 2024-04-26T04:50:12.513 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:50:12 smithi116 bash[31794]: dd1f1c263e39cebcaccfdd511a0ede6634c6eeda321b0af624487a37898404ac 2024-04-26T04:50:12.513 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:50:12 smithi116 systemd[1]: Started Ceph mgr.x for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:50:12.778 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:50:12 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:50:12.511+0000 7f20837381c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:50:12.779 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:50:12 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:50:12.673+0000 7f20837381c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:50:12.779 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:50:12 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:50:12.777+0000 7f20837381c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:50:13.485 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:50:13 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:50:13.109+0000 7f20837381c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:50:13.486 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:50:13 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:50:13.190+0000 7f20837381c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:50:13.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:13 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:13.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:13 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:13.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:13 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:13.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:13 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:13.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:13 smithi116 ceph-mon[29897]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:13.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:13 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:13.762 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:13 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:13.762 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:13 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:13.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:13 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:13.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:13 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:13.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:13 smithi012 ceph-mon[37567]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:13.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:13 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:13.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:13 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:13.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:13 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:13.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:13 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:13.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:13 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:13.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:13 smithi012 ceph-mon[26617]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:13.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:13 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:13.943 INFO:teuthology.orchestra.run.smithi012.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-26T04:50:13.943 INFO:teuthology.orchestra.run.smithi012.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-26T04:50:13.943 INFO:teuthology.orchestra.run.smithi012.stderr: stderr: 10+0 records in 2024-04-26T04:50:13.943 INFO:teuthology.orchestra.run.smithi012.stderr:10+0 records out 2024-04-26T04:50:13.944 INFO:teuthology.orchestra.run.smithi012.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0117224 s, 895 MB/s 2024-04-26T04:50:13.944 INFO:teuthology.orchestra.run.smithi012.stderr:--> Zapping successful for: 2024-04-26T04:50:14.504 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi012:vg_nvme/lv_4 2024-04-26T04:50:14.700 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:50:14 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:50:14.386+0000 7f20837381c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:50:14.700 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:50:14 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:50:14.549+0000 7f20837381c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:50:14.793 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:14 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:14.793 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:14 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:14.794 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:14 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:14.794 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:14 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:14.794 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:14 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:14.794 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:14 smithi012 ceph-mon[37567]: Reconfiguring mgr.y (unknown last config time)... 2024-04-26T04:50:14.794 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:14 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:50:14.794 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:14 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:50:14.794 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:14 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:14.794 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:14 smithi012 ceph-mon[37567]: Reconfiguring daemon mgr.y on smithi012 2024-04-26T04:50:14.794 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:14 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:14.794 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:14 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:14.794 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:14 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:14.795 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:14 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:14.795 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:14 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:14.795 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:14 smithi012 ceph-mon[26617]: Reconfiguring mgr.y (unknown last config time)... 2024-04-26T04:50:14.795 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:14 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:50:14.795 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:14 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:50:14.795 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:14 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:14.795 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:14 smithi012 ceph-mon[26617]: Reconfiguring daemon mgr.y on smithi012 2024-04-26T04:50:14.976 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:50:14 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:50:14.716+0000 7f20837381c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:50:14.976 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:50:14 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:50:14.975+0000 7f20837381c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:50:14.976 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:14 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:14.976 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:14 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:14.976 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:14 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:14.977 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:14 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:14.977 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:14 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:14.977 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:14 smithi116 ceph-mon[29897]: Reconfiguring mgr.y (unknown last config time)... 2024-04-26T04:50:14.977 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:14 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:50:14.977 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:14 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:50:14.977 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:14 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:14.977 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:14 smithi116 ceph-mon[29897]: Reconfiguring daemon mgr.y on smithi012 2024-04-26T04:50:15.241 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:50:15 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:50:15.064+0000 7f20837381c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:50:15.241 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:50:15 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:50:15.157+0000 7f20837381c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:50:15.580 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:50:15.714 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:50:15 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:50:15.446+0000 7f20837381c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:50:15.991 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:50:15 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:50:15.905+0000 7f20837381c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:50:15.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:15 smithi116 ceph-mon[29897]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:15.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:15 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:15.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:15 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:15.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:15 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:15.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:15 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:15.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:15 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:15.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:15 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:16.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:15 smithi012 ceph-mon[37567]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:16.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:15 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:16.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:15 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:16.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:15 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:16.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:15 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:16.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:15 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:16.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:15 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:16.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:15 smithi012 ceph-mon[26617]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:16.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:15 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:16.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:15 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:16.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:15 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:16.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:15 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:16.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:15 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:16.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:15 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:16.712 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:50:16 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:50:16.443+0000 7f20837381c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:50:16.712 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:50:16 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:50:16.535+0000 7f20837381c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:50:16.712 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:50:16 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:50:16.711+0000 7f20837381c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:50:17.256 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:50:17 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:50:17.000+0000 7f20837381c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:50:17.256 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:50:17 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:50:17.255+0000 7f20837381c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:50:17.611 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:50:17 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:50:17.485+0000 7f20837381c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:50:17.858 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:17 smithi012 ceph-mon[37567]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:17.858 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:17 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:17.858 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:17 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:50:17.858 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:17 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:50:17.858 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:17 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:17.859 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:17 smithi012 ceph-mon[26617]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:17.859 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:17 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:17.859 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:17 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:50:17.859 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:17 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:50:17.859 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:17 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:17.902 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:17 smithi116 ceph-mon[29897]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:17.902 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:17 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:17.902 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:17 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:50:17.902 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:17 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:50:17.902 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:17 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:17.902 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:50:17 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:50:17.613+0000 7f20837381c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:50:17.902 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:50:17 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:50:17.704+0000 7f20837381c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:50:17.902 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:50:17 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:50:17.814+0000 7f20837381c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:50:17.902 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:50:17 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:50:17.900+0000 7f20837381c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:50:18.873 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:18 smithi116 ceph-mon[29897]: from='client.14220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:50:18.874 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:50:18 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:50:18.683+0000 7f20837381c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:50:18.874 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:50:18 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:50:18.872+0000 7f20837381c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:50:18.906 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:18 smithi012 ceph-mon[26617]: from='client.14220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:50:18.906 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:18 smithi012 ceph-mon[37567]: from='client.14220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:50:19.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:19 smithi116 ceph-mon[29897]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:19.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:19 smithi116 ceph-mon[29897]: Standby manager daemon x started 2024-04-26T04:50:19.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:19 smithi116 ceph-mon[29897]: from='mgr.? 172.21.15.116:0/506617909' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:50:19.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:19 smithi116 ceph-mon[29897]: from='mgr.? 172.21.15.116:0/506617909' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:50:19.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:19 smithi116 ceph-mon[29897]: from='mgr.? 172.21.15.116:0/506617909' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:50:19.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:19 smithi116 ceph-mon[29897]: from='mgr.? 172.21.15.116:0/506617909' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:50:19.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:19 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/1022037355' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5f43d5c7-23d9-48a5-bc3c-a3a6ce3c004f"}]: dispatch 2024-04-26T04:50:19.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:19 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/1022037355' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5f43d5c7-23d9-48a5-bc3c-a3a6ce3c004f"}]': finished 2024-04-26T04:50:19.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:19 smithi116 ceph-mon[29897]: osdmap e5: 1 total, 0 up, 1 in 2024-04-26T04:50:19.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:19 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:50:20.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:19 smithi012 ceph-mon[26617]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:20.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:19 smithi012 ceph-mon[26617]: Standby manager daemon x started 2024-04-26T04:50:20.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:19 smithi012 ceph-mon[26617]: from='mgr.? 172.21.15.116:0/506617909' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:50:20.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:19 smithi012 ceph-mon[26617]: from='mgr.? 172.21.15.116:0/506617909' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:50:20.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:19 smithi012 ceph-mon[26617]: from='mgr.? 172.21.15.116:0/506617909' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:50:20.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:19 smithi012 ceph-mon[26617]: from='mgr.? 172.21.15.116:0/506617909' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:50:20.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:19 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1022037355' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5f43d5c7-23d9-48a5-bc3c-a3a6ce3c004f"}]: dispatch 2024-04-26T04:50:20.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:19 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1022037355' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5f43d5c7-23d9-48a5-bc3c-a3a6ce3c004f"}]': finished 2024-04-26T04:50:20.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:19 smithi012 ceph-mon[26617]: osdmap e5: 1 total, 0 up, 1 in 2024-04-26T04:50:20.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:19 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:50:20.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:19 smithi012 ceph-mon[37567]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:20.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:19 smithi012 ceph-mon[37567]: Standby manager daemon x started 2024-04-26T04:50:20.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:19 smithi012 ceph-mon[37567]: from='mgr.? 172.21.15.116:0/506617909' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:50:20.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:19 smithi012 ceph-mon[37567]: from='mgr.? 172.21.15.116:0/506617909' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:50:20.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:19 smithi012 ceph-mon[37567]: from='mgr.? 172.21.15.116:0/506617909' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:50:20.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:19 smithi012 ceph-mon[37567]: from='mgr.? 172.21.15.116:0/506617909' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:50:20.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:19 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/1022037355' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5f43d5c7-23d9-48a5-bc3c-a3a6ce3c004f"}]: dispatch 2024-04-26T04:50:20.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:19 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/1022037355' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5f43d5c7-23d9-48a5-bc3c-a3a6ce3c004f"}]': finished 2024-04-26T04:50:20.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:19 smithi012 ceph-mon[37567]: osdmap e5: 1 total, 0 up, 1 in 2024-04-26T04:50:20.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:19 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:50:20.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:20 smithi116 ceph-mon[29897]: mgrmap e14: y(active, since 63s), standbys: x 2024-04-26T04:50:20.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:20 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:50:20.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:20 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/2627327865' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:50:21.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:20 smithi012 ceph-mon[26617]: mgrmap e14: y(active, since 63s), standbys: x 2024-04-26T04:50:21.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:20 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:50:21.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:20 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/2627327865' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:50:21.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:20 smithi012 ceph-mon[37567]: mgrmap e14: y(active, since 63s), standbys: x 2024-04-26T04:50:21.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:20 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:50:21.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:20 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/2627327865' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:50:21.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:21 smithi116 ceph-mon[29897]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:22.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:21 smithi012 ceph-mon[26617]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:22.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:21 smithi012 ceph-mon[37567]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:23.881 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:23 smithi012 ceph-mon[26617]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:23.881 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:23 smithi012 ceph-mon[37567]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:23.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:23 smithi116 ceph-mon[29897]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:25.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:25 smithi116 ceph-mon[29897]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:26.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:25 smithi012 ceph-mon[26617]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:26.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:25 smithi012 ceph-mon[37567]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:26.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:26 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T04:50:26.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:26 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:26.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:26 smithi012 ceph-mon[26617]: Deploying daemon osd.0 on smithi012 2024-04-26T04:50:26.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:26 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T04:50:26.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:26 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:26.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:26 smithi012 ceph-mon[37567]: Deploying daemon osd.0 on smithi012 2024-04-26T04:50:26.893 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:26 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T04:50:26.893 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:26 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:26.893 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:26 smithi116 ceph-mon[29897]: Deploying daemon osd.0 on smithi012 2024-04-26T04:50:27.908 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:27 smithi116 ceph-mon[29897]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:27.979 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:27 smithi012 ceph-mon[26617]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:27.980 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:27 smithi012 ceph-mon[37567]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:29.955 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:29 smithi012 ceph-mon[26617]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:29.955 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:29 smithi012 ceph-mon[37567]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:29.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:29 smithi116 ceph-mon[29897]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:32.166 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:31 smithi012 ceph-mon[37567]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:32.166 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:31 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:32.166 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:31 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:32.166 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:31 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:32.166 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:31 smithi012 ceph-mon[26617]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:32.167 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:31 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:32.167 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:31 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:32.167 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:31 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:32.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:31 smithi116 ceph-mon[29897]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:32.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:31 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:32.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:31 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:32.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:31 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:33.875 INFO:teuthology.orchestra.run.smithi012.stdout:Created osd(s) 0 on host 'smithi012' 2024-04-26T04:50:34.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:33 smithi012 ceph-mon[26617]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:34.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:33 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:34.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:33 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:34.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:33 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:34.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:33 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:34.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:33 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:34.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:33 smithi012 ceph-mon[37567]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:34.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:33 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:34.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:33 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:34.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:33 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:34.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:33 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:34.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:33 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:34.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:33 smithi116 ceph-mon[29897]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:34.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:33 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:34.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:33 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:34.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:33 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:34.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:33 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:34.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:33 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:34.454 DEBUG:teuthology.orchestra.run.smithi012:osd.0> sudo journalctl -f -n 0 -u ceph-0b9350c6-0388-11ef-bc93-c7b262605968@osd.0.service 2024-04-26T04:50:34.457 INFO:tasks.cephadm:Deploying osd.1 on smithi012 with /dev/vg_nvme/lv_3... 2024-04-26T04:50:34.457 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-26T04:50:35.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:34 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:35.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:34 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:35.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:34 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:35.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:34 smithi116 ceph-mon[29897]: from='osd.0 [v2:172.21.15.12:6802/1077619568,v1:172.21.15.12:6803/1077619568]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T04:50:35.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:34 smithi116 ceph-mon[29897]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T04:50:35.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:34 smithi116 ceph-mon[29897]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:35.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:34 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:35.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:34 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:35.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:34 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:35.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:34 smithi012 ceph-mon[37567]: from='osd.0 [v2:172.21.15.12:6802/1077619568,v1:172.21.15.12:6803/1077619568]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T04:50:35.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:34 smithi012 ceph-mon[37567]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T04:50:35.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:34 smithi012 ceph-mon[37567]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:35.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:34 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:35.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:34 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:35.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:34 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:35.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:34 smithi012 ceph-mon[26617]: from='osd.0 [v2:172.21.15.12:6802/1077619568,v1:172.21.15.12:6803/1077619568]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T04:50:35.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:34 smithi012 ceph-mon[26617]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T04:50:35.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:34 smithi012 ceph-mon[26617]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:35.548 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:50:35.988 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:35 smithi012 ceph-mon[26617]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T04:50:35.989 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:35 smithi012 ceph-mon[26617]: osdmap e6: 1 total, 0 up, 1 in 2024-04-26T04:50:35.989 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:35 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:50:35.989 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:35 smithi012 ceph-mon[26617]: from='osd.0 [v2:172.21.15.12:6802/1077619568,v1:172.21.15.12:6803/1077619568]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-04-26T04:50:35.989 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:35 smithi012 ceph-mon[26617]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-04-26T04:50:35.989 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:35 smithi012 ceph-mon[37567]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T04:50:35.989 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:35 smithi012 ceph-mon[37567]: osdmap e6: 1 total, 0 up, 1 in 2024-04-26T04:50:35.989 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:35 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:50:35.989 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:35 smithi012 ceph-mon[37567]: from='osd.0 [v2:172.21.15.12:6802/1077619568,v1:172.21.15.12:6803/1077619568]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-04-26T04:50:35.989 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:35 smithi012 ceph-mon[37567]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-04-26T04:50:35.990 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 04:50:35 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0[41527]: 2024-04-26T04:50:35.864+0000 7f89a4a2d700 -1 osd.0 0 waiting for initial osdmap 2024-04-26T04:50:35.990 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 04:50:35 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0[41527]: 2024-04-26T04:50:35.867+0000 7f899e025700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:50:36.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:35 smithi116 ceph-mon[29897]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T04:50:36.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:35 smithi116 ceph-mon[29897]: osdmap e6: 1 total, 0 up, 1 in 2024-04-26T04:50:36.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:35 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:50:36.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:35 smithi116 ceph-mon[29897]: from='osd.0 [v2:172.21.15.12:6802/1077619568,v1:172.21.15.12:6803/1077619568]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-04-26T04:50:36.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:35 smithi116 ceph-mon[29897]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-04-26T04:50:37.095 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:36 smithi012 ceph-mon[37567]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2024-04-26T04:50:37.095 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:36 smithi012 ceph-mon[37567]: osdmap e7: 1 total, 0 up, 1 in 2024-04-26T04:50:37.095 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:36 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:50:37.095 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:36 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:50:37.095 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:36 smithi012 ceph-mon[37567]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:37.095 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:36 smithi012 ceph-mon[26617]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2024-04-26T04:50:37.095 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:36 smithi012 ceph-mon[26617]: osdmap e7: 1 total, 0 up, 1 in 2024-04-26T04:50:37.096 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:36 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:50:37.096 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:36 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:50:37.096 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:36 smithi012 ceph-mon[26617]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:37.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:36 smithi116 ceph-mon[29897]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2024-04-26T04:50:37.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:36 smithi116 ceph-mon[29897]: osdmap e7: 1 total, 0 up, 1 in 2024-04-26T04:50:37.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:36 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:50:37.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:36 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:50:37.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:36 smithi116 ceph-mon[29897]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:50:38.008 INFO:teuthology.orchestra.run.smithi012.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-26T04:50:38.008 INFO:teuthology.orchestra.run.smithi012.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-26T04:50:38.008 INFO:teuthology.orchestra.run.smithi012.stderr: stderr: 10+0 records in 2024-04-26T04:50:38.008 INFO:teuthology.orchestra.run.smithi012.stderr:10+0 records out 2024-04-26T04:50:38.008 INFO:teuthology.orchestra.run.smithi012.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0149446 s, 702 MB/s 2024-04-26T04:50:38.008 INFO:teuthology.orchestra.run.smithi012.stderr:--> Zapping successful for: 2024-04-26T04:50:38.031 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:37 smithi012 ceph-mon[26617]: purged_snaps scrub starts 2024-04-26T04:50:38.031 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:37 smithi012 ceph-mon[26617]: purged_snaps scrub ok 2024-04-26T04:50:38.031 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:37 smithi012 ceph-mon[26617]: osd.0 [v2:172.21.15.12:6802/1077619568,v1:172.21.15.12:6803/1077619568] boot 2024-04-26T04:50:38.031 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:37 smithi012 ceph-mon[26617]: osdmap e8: 1 total, 1 up, 1 in 2024-04-26T04:50:38.031 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:37 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:50:38.031 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:37 smithi012 ceph-mon[37567]: purged_snaps scrub starts 2024-04-26T04:50:38.031 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:37 smithi012 ceph-mon[37567]: purged_snaps scrub ok 2024-04-26T04:50:38.031 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:37 smithi012 ceph-mon[37567]: osd.0 [v2:172.21.15.12:6802/1077619568,v1:172.21.15.12:6803/1077619568] boot 2024-04-26T04:50:38.031 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:37 smithi012 ceph-mon[37567]: osdmap e8: 1 total, 1 up, 1 in 2024-04-26T04:50:38.031 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:37 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:50:38.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:37 smithi116 ceph-mon[29897]: purged_snaps scrub starts 2024-04-26T04:50:38.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:37 smithi116 ceph-mon[29897]: purged_snaps scrub ok 2024-04-26T04:50:38.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:37 smithi116 ceph-mon[29897]: osd.0 [v2:172.21.15.12:6802/1077619568,v1:172.21.15.12:6803/1077619568] boot 2024-04-26T04:50:38.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:37 smithi116 ceph-mon[29897]: osdmap e8: 1 total, 1 up, 1 in 2024-04-26T04:50:38.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:37 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:50:38.990 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:38 smithi012 ceph-mon[37567]: osdmap e9: 1 total, 1 up, 1 in 2024-04-26T04:50:38.990 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:38 smithi012 ceph-mon[37567]: pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:50:38.990 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:38 smithi012 ceph-mon[37567]: Detected new or changed devices on smithi012 2024-04-26T04:50:38.990 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:38 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:38.990 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:38 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:38.990 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:38 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:50:38.990 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:38 smithi012 ceph-mon[37567]: Adjusting osd_memory_target on smithi012 to 16069M 2024-04-26T04:50:38.990 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:38 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:38.990 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:38 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:38.991 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:38 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:38.991 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:38 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:38.991 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:38 smithi012 ceph-mon[26617]: osdmap e9: 1 total, 1 up, 1 in 2024-04-26T04:50:38.991 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:38 smithi012 ceph-mon[26617]: pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:50:38.991 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:38 smithi012 ceph-mon[26617]: Detected new or changed devices on smithi012 2024-04-26T04:50:38.991 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:38 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:38.991 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:38 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:38.991 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:38 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:50:38.991 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:38 smithi012 ceph-mon[26617]: Adjusting osd_memory_target on smithi012 to 16069M 2024-04-26T04:50:38.991 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:38 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:38.991 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:38 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:38.992 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:38 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:38.992 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:38 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:39.016 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi012:vg_nvme/lv_3 2024-04-26T04:50:39.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:38 smithi116 ceph-mon[29897]: osdmap e9: 1 total, 1 up, 1 in 2024-04-26T04:50:39.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:38 smithi116 ceph-mon[29897]: pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:50:39.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:38 smithi116 ceph-mon[29897]: Detected new or changed devices on smithi012 2024-04-26T04:50:39.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:38 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:39.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:38 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:39.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:38 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:50:39.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:38 smithi116 ceph-mon[29897]: Adjusting osd_memory_target on smithi012 to 16069M 2024-04-26T04:50:39.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:38 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:39.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:38 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:39.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:38 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:39.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:38 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:39.277 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:50:41.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:41 smithi116 ceph-mon[29897]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:50:41.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:41 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:50:41.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:41 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:50:41.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:41 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:42.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:41 smithi012 ceph-mon[26617]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:50:42.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:41 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:50:42.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:41 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:50:42.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:41 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:42.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:41 smithi012 ceph-mon[37567]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:50:42.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:41 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:50:42.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:41 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:50:42.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:41 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:42.879 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:42 smithi012 ceph-mon[26617]: from='client.24146 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:50:42.879 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:42 smithi012 ceph-mon[37567]: from='client.24146 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:50:42.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:42 smithi116 ceph-mon[29897]: from='client.24146 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:50:43.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:43 smithi116 ceph-mon[29897]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:50:43.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:43 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/1481391729' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "75471714-6e29-4e02-9c03-86ce919b1ff0"}]: dispatch 2024-04-26T04:50:43.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:43 smithi116 ceph-mon[29897]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "75471714-6e29-4e02-9c03-86ce919b1ff0"}]: dispatch 2024-04-26T04:50:43.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:43 smithi116 ceph-mon[29897]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "75471714-6e29-4e02-9c03-86ce919b1ff0"}]': finished 2024-04-26T04:50:43.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:43 smithi116 ceph-mon[29897]: osdmap e10: 2 total, 1 up, 2 in 2024-04-26T04:50:43.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:43 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:50:44.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:43 smithi012 ceph-mon[26617]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:50:44.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:43 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1481391729' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "75471714-6e29-4e02-9c03-86ce919b1ff0"}]: dispatch 2024-04-26T04:50:44.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:43 smithi012 ceph-mon[26617]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "75471714-6e29-4e02-9c03-86ce919b1ff0"}]: dispatch 2024-04-26T04:50:44.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:43 smithi012 ceph-mon[26617]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "75471714-6e29-4e02-9c03-86ce919b1ff0"}]': finished 2024-04-26T04:50:44.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:43 smithi012 ceph-mon[26617]: osdmap e10: 2 total, 1 up, 2 in 2024-04-26T04:50:44.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:43 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:50:44.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:43 smithi012 ceph-mon[37567]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:50:44.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:43 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/1481391729' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "75471714-6e29-4e02-9c03-86ce919b1ff0"}]: dispatch 2024-04-26T04:50:44.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:43 smithi012 ceph-mon[37567]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "75471714-6e29-4e02-9c03-86ce919b1ff0"}]: dispatch 2024-04-26T04:50:44.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:43 smithi012 ceph-mon[37567]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "75471714-6e29-4e02-9c03-86ce919b1ff0"}]': finished 2024-04-26T04:50:44.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:43 smithi012 ceph-mon[37567]: osdmap e10: 2 total, 1 up, 2 in 2024-04-26T04:50:44.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:43 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:50:44.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:44 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/987373459' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:50:45.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:44 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/987373459' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:50:45.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:44 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/987373459' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:50:45.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:45 smithi116 ceph-mon[29897]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:50:46.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:45 smithi012 ceph-mon[26617]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:50:46.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:45 smithi012 ceph-mon[37567]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:50:47.746 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:47 smithi012 ceph-mon[26617]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:50:47.746 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:47 smithi012 ceph-mon[37567]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:50:47.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:47 smithi116 ceph-mon[29897]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:50:49.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:49 smithi116 ceph-mon[29897]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:50:49.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:49 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T04:50:49.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:49 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:50.012 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:49 smithi012 ceph-mon[37567]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:50:50.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:49 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T04:50:50.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:49 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:50.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:49 smithi012 ceph-mon[26617]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:50:50.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:49 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T04:50:50.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:49 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:50.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:50 smithi116 ceph-mon[29897]: Deploying daemon osd.1 on smithi012 2024-04-26T04:50:51.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:50 smithi012 ceph-mon[26617]: Deploying daemon osd.1 on smithi012 2024-04-26T04:50:51.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:50 smithi012 ceph-mon[37567]: Deploying daemon osd.1 on smithi012 2024-04-26T04:50:51.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:51 smithi116 ceph-mon[29897]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:50:52.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:51 smithi012 ceph-mon[26617]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:50:52.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:51 smithi012 ceph-mon[37567]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:50:53.986 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:53 smithi012 ceph-mon[26617]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:50:53.986 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:53 smithi012 ceph-mon[37567]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:50:53.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:53 smithi116 ceph-mon[29897]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:50:55.806 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:55 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:55.806 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:55 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:55.806 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:55 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:55.806 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:55 smithi012 ceph-mon[26617]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:50:55.806 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:55 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:55.807 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:55 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:55.807 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:55 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:55.807 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:55 smithi012 ceph-mon[37567]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:50:55.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:55 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:55.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:55 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:55.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:55 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:55.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:55 smithi116 ceph-mon[29897]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:50:57.932 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:57 smithi012 ceph-mon[26617]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:50:57.932 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:57 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:57.932 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:57 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:57.932 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:57 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:57.932 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:57 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:57.932 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:57 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:57.932 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:57 smithi012 ceph-mon[26617]: from='osd.1 [v2:172.21.15.12:6810/726116340,v1:172.21.15.12:6811/726116340]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:50:57.933 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:57 smithi012 ceph-mon[37567]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:50:57.933 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:57 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:57.933 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:57 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:57.933 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:57 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:57.933 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:57 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:57.933 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:57 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:57.934 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:57 smithi012 ceph-mon[37567]: from='osd.1 [v2:172.21.15.12:6810/726116340,v1:172.21.15.12:6811/726116340]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:50:57.976 INFO:teuthology.orchestra.run.smithi012.stdout:Created osd(s) 1 on host 'smithi012' 2024-04-26T04:50:58.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:57 smithi116 ceph-mon[29897]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:50:58.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:57 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:58.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:57 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:58.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:57 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:58.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:57 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:58.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:57 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:58.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:57 smithi116 ceph-mon[29897]: from='osd.1 [v2:172.21.15.12:6810/726116340,v1:172.21.15.12:6811/726116340]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:50:58.543 DEBUG:teuthology.orchestra.run.smithi012:osd.1> sudo journalctl -f -n 0 -u ceph-0b9350c6-0388-11ef-bc93-c7b262605968@osd.1.service 2024-04-26T04:50:58.546 INFO:tasks.cephadm:Deploying osd.2 on smithi012 with /dev/vg_nvme/lv_2... 2024-04-26T04:50:58.546 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-26T04:50:59.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:58 smithi116 ceph-mon[29897]: from='osd.1 [v2:172.21.15.12:6810/726116340,v1:172.21.15.12:6811/726116340]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T04:50:59.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:58 smithi116 ceph-mon[29897]: osdmap e11: 2 total, 1 up, 2 in 2024-04-26T04:50:59.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:58 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:50:59.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:58 smithi116 ceph-mon[29897]: from='osd.1 [v2:172.21.15.12:6810/726116340,v1:172.21.15.12:6811/726116340]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-04-26T04:50:59.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:58 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:59.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:58 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:59.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:58 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:59.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:58 smithi116 ceph-mon[29897]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:50:59.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:58 smithi012 ceph-mon[26617]: from='osd.1 [v2:172.21.15.12:6810/726116340,v1:172.21.15.12:6811/726116340]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T04:50:59.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:58 smithi012 ceph-mon[26617]: osdmap e11: 2 total, 1 up, 2 in 2024-04-26T04:50:59.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:58 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:50:59.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:58 smithi012 ceph-mon[26617]: from='osd.1 [v2:172.21.15.12:6810/726116340,v1:172.21.15.12:6811/726116340]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-04-26T04:50:59.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:58 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:59.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:58 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:59.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:58 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:59.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:58 smithi012 ceph-mon[26617]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:50:59.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:58 smithi012 ceph-mon[37567]: from='osd.1 [v2:172.21.15.12:6810/726116340,v1:172.21.15.12:6811/726116340]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T04:50:59.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:58 smithi012 ceph-mon[37567]: osdmap e11: 2 total, 1 up, 2 in 2024-04-26T04:50:59.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:58 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:50:59.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:58 smithi012 ceph-mon[37567]: from='osd.1 [v2:172.21.15.12:6810/726116340,v1:172.21.15.12:6811/726116340]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-04-26T04:50:59.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:58 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:59.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:58 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:50:59.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:58 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:59.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:58 smithi012 ceph-mon[37567]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:50:59.264 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 04:50:58 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1[47231]: 2024-04-26T04:50:58.850+0000 7fb5a8794700 -1 osd.1 0 waiting for initial osdmap 2024-04-26T04:50:59.264 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 04:50:58 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1[47231]: 2024-04-26T04:50:58.853+0000 7fb5a5593700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:50:59.340 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:50:59.983 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:59 smithi012 ceph-mon[26617]: from='osd.1 [v2:172.21.15.12:6810/726116340,v1:172.21.15.12:6811/726116340]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2024-04-26T04:50:59.983 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:59 smithi012 ceph-mon[26617]: osdmap e12: 2 total, 1 up, 2 in 2024-04-26T04:50:59.983 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:59 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:50:59.983 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:50:59 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:50:59.983 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:59 smithi012 ceph-mon[37567]: from='osd.1 [v2:172.21.15.12:6810/726116340,v1:172.21.15.12:6811/726116340]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2024-04-26T04:50:59.983 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:59 smithi012 ceph-mon[37567]: osdmap e12: 2 total, 1 up, 2 in 2024-04-26T04:50:59.983 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:59 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:50:59.984 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:50:59 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:51:00.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:59 smithi116 ceph-mon[29897]: from='osd.1 [v2:172.21.15.12:6810/726116340,v1:172.21.15.12:6811/726116340]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2024-04-26T04:51:00.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:59 smithi116 ceph-mon[29897]: osdmap e12: 2 total, 1 up, 2 in 2024-04-26T04:51:00.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:59 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:51:00.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:50:59 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:51:01.198 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:00 smithi012 ceph-mon[26617]: purged_snaps scrub starts 2024-04-26T04:51:01.199 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:00 smithi012 ceph-mon[26617]: purged_snaps scrub ok 2024-04-26T04:51:01.199 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:00 smithi012 ceph-mon[26617]: osd.1 [v2:172.21.15.12:6810/726116340,v1:172.21.15.12:6811/726116340] boot 2024-04-26T04:51:01.199 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:00 smithi012 ceph-mon[26617]: osdmap e13: 2 total, 2 up, 2 in 2024-04-26T04:51:01.199 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:00 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:51:01.199 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:00 smithi012 ceph-mon[26617]: pgmap v55: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:01.199 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:00 smithi012 ceph-mon[37567]: purged_snaps scrub starts 2024-04-26T04:51:01.199 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:00 smithi012 ceph-mon[37567]: purged_snaps scrub ok 2024-04-26T04:51:01.199 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:00 smithi012 ceph-mon[37567]: osd.1 [v2:172.21.15.12:6810/726116340,v1:172.21.15.12:6811/726116340] boot 2024-04-26T04:51:01.199 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:00 smithi012 ceph-mon[37567]: osdmap e13: 2 total, 2 up, 2 in 2024-04-26T04:51:01.200 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:00 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:51:01.200 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:00 smithi012 ceph-mon[37567]: pgmap v55: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:01.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:00 smithi116 ceph-mon[29897]: purged_snaps scrub starts 2024-04-26T04:51:01.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:00 smithi116 ceph-mon[29897]: purged_snaps scrub ok 2024-04-26T04:51:01.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:00 smithi116 ceph-mon[29897]: osd.1 [v2:172.21.15.12:6810/726116340,v1:172.21.15.12:6811/726116340] boot 2024-04-26T04:51:01.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:00 smithi116 ceph-mon[29897]: osdmap e13: 2 total, 2 up, 2 in 2024-04-26T04:51:01.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:00 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:51:01.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:00 smithi116 ceph-mon[29897]: pgmap v55: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:51:02.171 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:01 smithi012 ceph-mon[26617]: osdmap e14: 2 total, 2 up, 2 in 2024-04-26T04:51:02.171 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:01 smithi012 ceph-mon[37567]: osdmap e14: 2 total, 2 up, 2 in 2024-04-26T04:51:02.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:01 smithi116 ceph-mon[29897]: osdmap e14: 2 total, 2 up, 2 in 2024-04-26T04:51:02.644 INFO:teuthology.orchestra.run.smithi012.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-26T04:51:02.644 INFO:teuthology.orchestra.run.smithi012.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-26T04:51:02.644 INFO:teuthology.orchestra.run.smithi012.stderr: stderr: 10+0 records in 2024-04-26T04:51:02.645 INFO:teuthology.orchestra.run.smithi012.stderr:10+0 records out 2024-04-26T04:51:02.645 INFO:teuthology.orchestra.run.smithi012.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0108944 s, 962 MB/s 2024-04-26T04:51:02.645 INFO:teuthology.orchestra.run.smithi012.stderr:--> Zapping successful for: 2024-04-26T04:51:03.239 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:03 smithi012 ceph-mon[26617]: Detected new or changed devices on smithi012 2024-04-26T04:51:03.239 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:03 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:03.240 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:03 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:03.240 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:03 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:51:03.240 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:03 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:51:03.240 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:03 smithi012 ceph-mon[26617]: Adjusting osd_memory_target on smithi012 to 8034M 2024-04-26T04:51:03.240 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:03 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:03.240 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:03 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:03.240 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:03 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:03.240 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:03 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:03.240 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:03 smithi012 ceph-mon[26617]: pgmap v57: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:03.240 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:03 smithi012 ceph-mon[37567]: Detected new or changed devices on smithi012 2024-04-26T04:51:03.241 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:03 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:03.241 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:03 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:03.241 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:03 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:51:03.241 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:03 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:51:03.241 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:03 smithi012 ceph-mon[37567]: Adjusting osd_memory_target on smithi012 to 8034M 2024-04-26T04:51:03.241 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:03 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:03.241 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:03 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:03.241 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:03 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:03.241 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:03 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:03.241 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:03 smithi012 ceph-mon[37567]: pgmap v57: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:03.242 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi012:vg_nvme/lv_2 2024-04-26T04:51:03.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:03 smithi116 ceph-mon[29897]: Detected new or changed devices on smithi012 2024-04-26T04:51:03.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:03 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:03.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:03 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:03.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:03 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:51:03.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:03 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:51:03.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:03 smithi116 ceph-mon[29897]: Adjusting osd_memory_target on smithi012 to 8034M 2024-04-26T04:51:03.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:03 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:03.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:03 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:03.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:03 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:03.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:03 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:03.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:03 smithi116 ceph-mon[29897]: pgmap v57: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:03.507 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:51:05.909 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:05 smithi012 ceph-mon[26617]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:05.909 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:05 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:51:05.909 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:05 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:51:05.909 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:05 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:05.909 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:05 smithi012 ceph-mon[37567]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:05.909 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:05 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:51:05.909 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:05 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:51:05.910 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:05 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:05.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:05 smithi116 ceph-mon[29897]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:05.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:05 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:51:05.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:05 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:51:05.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:05 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:06.908 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:06 smithi012 ceph-mon[26617]: from='client.24173 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:06.908 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:06 smithi012 ceph-mon[37567]: from='client.24173 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:06.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:06 smithi116 ceph-mon[29897]: from='client.24173 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:07.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:07 smithi116 ceph-mon[29897]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:08.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:07 smithi012 ceph-mon[26617]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:08.012 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:07 smithi012 ceph-mon[37567]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:08.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:08 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/186379138' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4b402201-4f28-4bc1-9cf0-0384090fb158"}]: dispatch 2024-04-26T04:51:08.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:08 smithi116 ceph-mon[29897]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4b402201-4f28-4bc1-9cf0-0384090fb158"}]: dispatch 2024-04-26T04:51:08.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:08 smithi116 ceph-mon[29897]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4b402201-4f28-4bc1-9cf0-0384090fb158"}]': finished 2024-04-26T04:51:08.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:08 smithi116 ceph-mon[29897]: osdmap e15: 3 total, 2 up, 3 in 2024-04-26T04:51:08.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:08 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:51:08.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:08 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/4112240409' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:51:09.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:08 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/186379138' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4b402201-4f28-4bc1-9cf0-0384090fb158"}]: dispatch 2024-04-26T04:51:09.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:08 smithi012 ceph-mon[37567]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4b402201-4f28-4bc1-9cf0-0384090fb158"}]: dispatch 2024-04-26T04:51:09.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:08 smithi012 ceph-mon[37567]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4b402201-4f28-4bc1-9cf0-0384090fb158"}]': finished 2024-04-26T04:51:09.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:08 smithi012 ceph-mon[37567]: osdmap e15: 3 total, 2 up, 3 in 2024-04-26T04:51:09.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:08 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:51:09.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:08 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/4112240409' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:51:09.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:08 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/186379138' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4b402201-4f28-4bc1-9cf0-0384090fb158"}]: dispatch 2024-04-26T04:51:09.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:08 smithi012 ceph-mon[26617]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4b402201-4f28-4bc1-9cf0-0384090fb158"}]: dispatch 2024-04-26T04:51:09.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:08 smithi012 ceph-mon[26617]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4b402201-4f28-4bc1-9cf0-0384090fb158"}]': finished 2024-04-26T04:51:09.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:08 smithi012 ceph-mon[26617]: osdmap e15: 3 total, 2 up, 3 in 2024-04-26T04:51:09.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:08 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:51:09.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:08 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/4112240409' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:51:09.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:09 smithi116 ceph-mon[29897]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:10.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:09 smithi012 ceph-mon[26617]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:10.012 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:09 smithi012 ceph-mon[37567]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:11.988 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:11 smithi012 ceph-mon[26617]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:11.988 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:11 smithi012 ceph-mon[37567]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:11.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:11 smithi116 ceph-mon[29897]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:13.966 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:13 smithi012 ceph-mon[26617]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:13.967 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:13 smithi012 ceph-mon[37567]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:13.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:13 smithi116 ceph-mon[29897]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:15.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:14 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T04:51:15.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:14 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:15.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:14 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T04:51:15.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:14 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:15.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:14 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T04:51:15.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:14 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:15.967 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:15 smithi012 ceph-mon[37567]: Deploying daemon osd.2 on smithi012 2024-04-26T04:51:15.967 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:15 smithi012 ceph-mon[37567]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:15.967 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:15 smithi012 ceph-mon[26617]: Deploying daemon osd.2 on smithi012 2024-04-26T04:51:15.967 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:15 smithi012 ceph-mon[26617]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:16.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:15 smithi116 ceph-mon[29897]: Deploying daemon osd.2 on smithi012 2024-04-26T04:51:16.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:15 smithi116 ceph-mon[29897]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:18.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:17 smithi012 ceph-mon[26617]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:18.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:17 smithi012 ceph-mon[37567]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:18.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:17 smithi116 ceph-mon[29897]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:20.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:19 smithi012 ceph-mon[26617]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:20.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:19 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:20.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:19 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:20.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:19 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:20.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:19 smithi012 ceph-mon[37567]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:20.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:19 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:20.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:19 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:20.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:19 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:20.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:19 smithi116 ceph-mon[29897]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:20.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:19 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:20.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:19 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:20.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:19 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:21.965 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:21 smithi012 ceph-mon[26617]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:21.965 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:21 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:21.965 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:21 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:21.965 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:21 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:21.965 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:21 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:21.965 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:21 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:21.966 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:21 smithi012 ceph-mon[37567]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:21.966 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:21 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:21.966 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:21 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:21.966 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:21 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:21.966 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:21 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:21.966 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:21 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:22.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:21 smithi116 ceph-mon[29897]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:22.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:21 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:22.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:21 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:22.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:21 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:22.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:21 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:22.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:21 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:22.330 INFO:teuthology.orchestra.run.smithi012.stdout:Created osd(s) 2 on host 'smithi012' 2024-04-26T04:51:22.953 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:22 smithi012 ceph-mon[37567]: from='osd.2 [v2:172.21.15.12:6818/4118496923,v1:172.21.15.12:6819/4118496923]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:51:22.953 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:22 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:22.953 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:22 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:22.953 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:22 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:22.953 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:22 smithi012 ceph-mon[26617]: from='osd.2 [v2:172.21.15.12:6818/4118496923,v1:172.21.15.12:6819/4118496923]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:51:22.953 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:22 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:22.954 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:22 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:22.954 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:22 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:22.984 DEBUG:teuthology.orchestra.run.smithi012:osd.2> sudo journalctl -f -n 0 -u ceph-0b9350c6-0388-11ef-bc93-c7b262605968@osd.2.service 2024-04-26T04:51:22.986 INFO:tasks.cephadm:Deploying osd.3 on smithi012 with /dev/vg_nvme/lv_1... 2024-04-26T04:51:22.987 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-26T04:51:23.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:22 smithi116 ceph-mon[29897]: from='osd.2 [v2:172.21.15.12:6818/4118496923,v1:172.21.15.12:6819/4118496923]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:51:23.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:22 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:23.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:22 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:23.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:22 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:23.951 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:23 smithi012 ceph-mon[26617]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:23.951 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:23 smithi012 ceph-mon[26617]: from='osd.2 [v2:172.21.15.12:6818/4118496923,v1:172.21.15.12:6819/4118496923]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T04:51:23.952 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:23 smithi012 ceph-mon[26617]: osdmap e16: 3 total, 2 up, 3 in 2024-04-26T04:51:23.952 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:23 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:51:23.952 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:23 smithi012 ceph-mon[26617]: from='osd.2 [v2:172.21.15.12:6818/4118496923,v1:172.21.15.12:6819/4118496923]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-04-26T04:51:23.952 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:23 smithi012 ceph-mon[37567]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:23.952 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:23 smithi012 ceph-mon[37567]: from='osd.2 [v2:172.21.15.12:6818/4118496923,v1:172.21.15.12:6819/4118496923]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T04:51:23.952 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:23 smithi012 ceph-mon[37567]: osdmap e16: 3 total, 2 up, 3 in 2024-04-26T04:51:23.952 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:23 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:51:23.952 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:23 smithi012 ceph-mon[37567]: from='osd.2 [v2:172.21.15.12:6818/4118496923,v1:172.21.15.12:6819/4118496923]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-04-26T04:51:23.952 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 04:51:23 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2[52883]: 2024-04-26T04:51:23.767+0000 7f0e02a61700 -1 osd.2 0 waiting for initial osdmap 2024-04-26T04:51:23.953 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 04:51:23 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2[52883]: 2024-04-26T04:51:23.772+0000 7f0dfa856700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:51:24.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:23 smithi116 ceph-mon[29897]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:24.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:23 smithi116 ceph-mon[29897]: from='osd.2 [v2:172.21.15.12:6818/4118496923,v1:172.21.15.12:6819/4118496923]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T04:51:24.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:23 smithi116 ceph-mon[29897]: osdmap e16: 3 total, 2 up, 3 in 2024-04-26T04:51:24.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:23 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:51:24.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:23 smithi116 ceph-mon[29897]: from='osd.2 [v2:172.21.15.12:6818/4118496923,v1:172.21.15.12:6819/4118496923]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-04-26T04:51:24.471 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:51:24.850 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:24 smithi012 ceph-mon[26617]: from='osd.2 [v2:172.21.15.12:6818/4118496923,v1:172.21.15.12:6819/4118496923]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2024-04-26T04:51:24.850 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:24 smithi012 ceph-mon[26617]: osdmap e17: 3 total, 2 up, 3 in 2024-04-26T04:51:24.850 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:24 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:51:24.850 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:24 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:51:24.851 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:24 smithi012 ceph-mon[37567]: from='osd.2 [v2:172.21.15.12:6818/4118496923,v1:172.21.15.12:6819/4118496923]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2024-04-26T04:51:24.851 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:24 smithi012 ceph-mon[37567]: osdmap e17: 3 total, 2 up, 3 in 2024-04-26T04:51:24.851 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:24 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:51:24.851 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:24 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:51:25.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:24 smithi116 ceph-mon[29897]: from='osd.2 [v2:172.21.15.12:6818/4118496923,v1:172.21.15.12:6819/4118496923]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2024-04-26T04:51:25.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:24 smithi116 ceph-mon[29897]: osdmap e17: 3 total, 2 up, 3 in 2024-04-26T04:51:25.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:24 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:51:25.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:24 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:51:26.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:25 smithi012 ceph-mon[26617]: purged_snaps scrub starts 2024-04-26T04:51:26.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:25 smithi012 ceph-mon[26617]: purged_snaps scrub ok 2024-04-26T04:51:26.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:25 smithi012 ceph-mon[26617]: pgmap v71: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:26.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:25 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:51:26.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:25 smithi012 ceph-mon[26617]: osd.2 [v2:172.21.15.12:6818/4118496923,v1:172.21.15.12:6819/4118496923] boot 2024-04-26T04:51:26.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:25 smithi012 ceph-mon[26617]: osdmap e18: 3 total, 3 up, 3 in 2024-04-26T04:51:26.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:25 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:51:26.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:25 smithi012 ceph-mon[37567]: purged_snaps scrub starts 2024-04-26T04:51:26.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:25 smithi012 ceph-mon[37567]: purged_snaps scrub ok 2024-04-26T04:51:26.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:25 smithi012 ceph-mon[37567]: pgmap v71: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:26.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:25 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:51:26.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:25 smithi012 ceph-mon[37567]: osd.2 [v2:172.21.15.12:6818/4118496923,v1:172.21.15.12:6819/4118496923] boot 2024-04-26T04:51:26.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:25 smithi012 ceph-mon[37567]: osdmap e18: 3 total, 3 up, 3 in 2024-04-26T04:51:26.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:25 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:51:26.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:25 smithi116 ceph-mon[29897]: purged_snaps scrub starts 2024-04-26T04:51:26.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:25 smithi116 ceph-mon[29897]: purged_snaps scrub ok 2024-04-26T04:51:26.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:25 smithi116 ceph-mon[29897]: pgmap v71: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:51:26.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:25 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:51:26.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:25 smithi116 ceph-mon[29897]: osd.2 [v2:172.21.15.12:6818/4118496923,v1:172.21.15.12:6819/4118496923] boot 2024-04-26T04:51:26.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:25 smithi116 ceph-mon[29897]: osdmap e18: 3 total, 3 up, 3 in 2024-04-26T04:51:26.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:25 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:51:26.897 INFO:teuthology.orchestra.run.smithi012.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-26T04:51:26.897 INFO:teuthology.orchestra.run.smithi012.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-26T04:51:26.897 INFO:teuthology.orchestra.run.smithi012.stderr: stderr: 10+0 records in 2024-04-26T04:51:26.897 INFO:teuthology.orchestra.run.smithi012.stderr:10+0 records out 2024-04-26T04:51:26.897 INFO:teuthology.orchestra.run.smithi012.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0167087 s, 628 MB/s 2024-04-26T04:51:26.897 INFO:teuthology.orchestra.run.smithi012.stderr:--> Zapping successful for: 2024-04-26T04:51:27.215 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:27 smithi012 ceph-mon[26617]: osdmap e19: 3 total, 3 up, 3 in 2024-04-26T04:51:27.215 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:27 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:27.215 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:27 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:27.215 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:27 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:51:27.215 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:27 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:51:27.215 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:27 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:51:27.215 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:27 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:27.215 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:27 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:27.215 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:27 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:27.215 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:27 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:27.216 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:27 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-26T04:51:27.216 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:27 smithi012 ceph-mon[37567]: osdmap e19: 3 total, 3 up, 3 in 2024-04-26T04:51:27.216 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:27 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:27.216 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:27 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:27.216 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:27 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:51:27.216 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:27 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:51:27.216 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:27 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:51:27.216 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:27 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:27.216 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:27 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:27.216 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:27 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:27.216 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:27 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:27.216 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:27 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-26T04:51:27.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:27 smithi116 ceph-mon[29897]: osdmap e19: 3 total, 3 up, 3 in 2024-04-26T04:51:27.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:27 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:27.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:27 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:27.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:27 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:51:27.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:27 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:51:27.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:27 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:51:27.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:27 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:27.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:27 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:27.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:27 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:27.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:27 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:27.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:27 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-26T04:51:27.517 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi012:vg_nvme/lv_1 2024-04-26T04:51:27.780 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:51:28.236 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:28 smithi012 ceph-mon[26617]: Detected new or changed devices on smithi012 2024-04-26T04:51:28.236 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:28 smithi012 ceph-mon[26617]: Adjusting osd_memory_target on smithi012 to 5356M 2024-04-26T04:51:28.236 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:28 smithi012 ceph-mon[26617]: pgmap v74: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:51:28.236 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:28 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-26T04:51:28.236 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:28 smithi012 ceph-mon[26617]: osdmap e20: 3 total, 3 up, 3 in 2024-04-26T04:51:28.236 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:28 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:51:28.236 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:28 smithi012 ceph-mon[37567]: Detected new or changed devices on smithi012 2024-04-26T04:51:28.236 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:28 smithi012 ceph-mon[37567]: Adjusting osd_memory_target on smithi012 to 5356M 2024-04-26T04:51:28.237 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:28 smithi012 ceph-mon[37567]: pgmap v74: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:51:28.237 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:28 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-26T04:51:28.237 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:28 smithi012 ceph-mon[37567]: osdmap e20: 3 total, 3 up, 3 in 2024-04-26T04:51:28.237 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:28 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:51:28.393 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:28 smithi116 ceph-mon[29897]: Detected new or changed devices on smithi012 2024-04-26T04:51:28.393 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:28 smithi116 ceph-mon[29897]: Adjusting osd_memory_target on smithi012 to 5356M 2024-04-26T04:51:28.393 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:28 smithi116 ceph-mon[29897]: pgmap v74: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:51:28.393 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:28 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-26T04:51:28.393 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:28 smithi116 ceph-mon[29897]: osdmap e20: 3 total, 3 up, 3 in 2024-04-26T04:51:28.393 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:28 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:51:28.854 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 04:51:28 smithi012 sudo[56808]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T04:51:28.854 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 04:51:28 smithi012 sudo[56808]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:51:28.854 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 04:51:28 smithi012 sudo[56808]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:51:28.854 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 04:51:28 smithi012 sudo[56808]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:51:29.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:29 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:51:29.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:29 smithi012 ceph-mon[26617]: osdmap e21: 3 total, 3 up, 3 in 2024-04-26T04:51:29.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:29 smithi012 ceph-mon[26617]: pgmap v77: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:51:29.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:29 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:51:29.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:29 smithi012 ceph-mon[37567]: osdmap e21: 3 total, 3 up, 3 in 2024-04-26T04:51:29.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:29 smithi012 ceph-mon[37567]: pgmap v77: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:51:29.514 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 04:51:29 smithi012 sudo[56838]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T04:51:29.514 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 04:51:29 smithi012 sudo[56838]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:51:29.514 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 04:51:29 smithi012 sudo[56838]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:51:29.514 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 04:51:29 smithi012 sudo[56838]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:51:29.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:29 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:51:29.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:29 smithi116 ceph-mon[29897]: osdmap e21: 3 total, 3 up, 3 in 2024-04-26T04:51:29.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:29 smithi116 ceph-mon[29897]: pgmap v77: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:51:30.118 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 04:51:29 smithi012 sudo[56928]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T04:51:30.118 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 04:51:29 smithi012 sudo[56928]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:51:30.118 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 04:51:29 smithi012 sudo[56928]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:51:30.118 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 04:51:29 smithi012 sudo[56928]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:51:30.388 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 04:51:30 smithi012 sudo[57001]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T04:51:30.388 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 04:51:30 smithi012 sudo[57001]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:51:30.388 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 04:51:30 smithi012 sudo[57001]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:51:30.388 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 04:51:30 smithi012 sudo[57001]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:51:30.388 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:30 smithi012 ceph-mon[26617]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-26T04:51:30.388 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:30 smithi012 ceph-mon[26617]: osdmap e22: 3 total, 3 up, 3 in 2024-04-26T04:51:30.389 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:30 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:30.389 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:30 smithi012 ceph-mon[26617]: from='client.14307 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:30.389 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:30 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:51:30.389 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:30 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:51:30.389 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:30 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:30.389 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:30 smithi012 ceph-mon[37567]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-26T04:51:30.389 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:30 smithi012 ceph-mon[37567]: osdmap e22: 3 total, 3 up, 3 in 2024-04-26T04:51:30.389 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:30 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:30.389 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:30 smithi012 ceph-mon[37567]: from='client.14307 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:30.389 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:30 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:51:30.389 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:30 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:51:30.389 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:30 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:30.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:30 smithi116 ceph-mon[29897]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-26T04:51:30.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:30 smithi116 ceph-mon[29897]: osdmap e22: 3 total, 3 up, 3 in 2024-04-26T04:51:30.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:30 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:30.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:30 smithi116 ceph-mon[29897]: from='client.14307 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:30.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:30 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:51:30.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:30 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:51:30.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:30 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:30.942 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 04:51:30 smithi012 sudo[57016]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T04:51:30.943 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 04:51:30 smithi012 sudo[57016]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:51:30.943 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 04:51:30 smithi012 sudo[57016]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:51:30.943 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 04:51:30 smithi012 sudo[57016]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:51:31.222 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 04:51:31 smithi012 sudo[57042]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T04:51:31.222 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 04:51:31 smithi012 sudo[57042]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:51:31.222 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 04:51:31 smithi012 sudo[57042]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:51:31.222 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 04:51:31 smithi012 sudo[57042]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:51:32.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:31 smithi012 sudo[57057]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T04:51:32.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:31 smithi012 sudo[57057]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:51:32.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:31 smithi012 sudo[57057]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:51:32.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:31 smithi012 sudo[57057]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:51:32.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:32 smithi012 ceph-mon[37567]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:51:32.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:32 smithi012 sudo[57098]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T04:51:32.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:32 smithi012 sudo[57098]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:51:32.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:32 smithi012 sudo[57098]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:51:32.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:32 smithi012 sudo[57098]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:51:32.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:32 smithi012 ceph-mon[26617]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:51:32.561 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:32 smithi116 sudo[32510]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T04:51:32.562 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:32 smithi116 sudo[32510]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:51:32.562 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:32 smithi116 sudo[32510]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:51:32.562 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:32 smithi116 sudo[32510]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:51:32.562 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:32 smithi116 sudo[32513]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T04:51:32.562 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:32 smithi116 sudo[32513]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:51:32.562 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:32 smithi116 sudo[32513]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:51:32.562 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:32 smithi116 sudo[32513]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:51:32.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:32 smithi116 ceph-mon[29897]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:51:33.494 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:33 smithi012 sudo[57134]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T04:51:33.494 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:33 smithi012 sudo[57134]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:51:33.495 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:33 smithi012 sudo[57134]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:51:33.495 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:33 smithi012 sudo[57134]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:51:33.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[26617]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:51:33.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[26617]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:51:33.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:51:33.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/4087963824' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "54bf03b9-b2e6-4230-8dcd-d62af8853c36"}]: dispatch 2024-04-26T04:51:33.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:51:33.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:51:33.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[26617]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "54bf03b9-b2e6-4230-8dcd-d62af8853c36"}]: dispatch 2024-04-26T04:51:33.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[26617]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:51:33.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:51:33.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:51:33.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:51:33.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[26617]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:51:33.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[26617]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "54bf03b9-b2e6-4230-8dcd-d62af8853c36"}]': finished 2024-04-26T04:51:33.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[26617]: osdmap e23: 4 total, 3 up, 4 in 2024-04-26T04:51:33.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:51:33.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[26617]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:51:33.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:33.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:51:33.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:51:33.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:51:33.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:33 smithi012 sudo[57137]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T04:51:33.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:33 smithi012 sudo[57137]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:51:33.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:33 smithi012 sudo[57137]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:51:33.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:33 smithi012 sudo[57137]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:51:33.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[37567]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:51:33.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[37567]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:51:33.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:51:33.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/4087963824' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "54bf03b9-b2e6-4230-8dcd-d62af8853c36"}]: dispatch 2024-04-26T04:51:33.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:51:33.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:51:33.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[37567]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "54bf03b9-b2e6-4230-8dcd-d62af8853c36"}]: dispatch 2024-04-26T04:51:33.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[37567]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:51:33.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:51:33.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:51:33.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:51:33.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[37567]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:51:33.766 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[37567]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "54bf03b9-b2e6-4230-8dcd-d62af8853c36"}]': finished 2024-04-26T04:51:33.766 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[37567]: osdmap e23: 4 total, 3 up, 4 in 2024-04-26T04:51:33.766 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:51:33.766 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[37567]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:51:33.766 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:33.766 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:51:33.766 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:51:33.766 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:33 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:51:33.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:33 smithi116 ceph-mon[29897]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:51:33.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:33 smithi116 ceph-mon[29897]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:51:33.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:33 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:51:33.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:33 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/4087963824' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "54bf03b9-b2e6-4230-8dcd-d62af8853c36"}]: dispatch 2024-04-26T04:51:33.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:33 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:51:33.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:33 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:51:33.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:33 smithi116 ceph-mon[29897]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "54bf03b9-b2e6-4230-8dcd-d62af8853c36"}]: dispatch 2024-04-26T04:51:33.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:33 smithi116 ceph-mon[29897]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:51:33.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:33 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:51:33.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:33 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:51:33.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:33 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:51:33.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:33 smithi116 ceph-mon[29897]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:51:33.993 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:33 smithi116 ceph-mon[29897]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "54bf03b9-b2e6-4230-8dcd-d62af8853c36"}]': finished 2024-04-26T04:51:33.993 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:33 smithi116 ceph-mon[29897]: osdmap e23: 4 total, 3 up, 4 in 2024-04-26T04:51:33.993 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:33 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:51:33.993 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:33 smithi116 ceph-mon[29897]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:51:33.993 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:33 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:33.993 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:33 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:51:33.993 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:33 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:51:33.993 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:33 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:51:34.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:34 smithi012 ceph-mon[26617]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:51:34.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:34 smithi012 ceph-mon[26617]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:51:34.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:34 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/2615317459' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:51:34.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:34 smithi012 ceph-mon[37567]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:51:34.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:34 smithi012 ceph-mon[37567]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:51:34.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:34 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/2615317459' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:51:34.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:34 smithi116 ceph-mon[29897]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:51:34.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:34 smithi116 ceph-mon[29897]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:51:34.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:34 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/2615317459' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:51:35.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:35 smithi012 ceph-mon[26617]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:51:35.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:35 smithi012 ceph-mon[37567]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:51:35.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:35 smithi116 ceph-mon[29897]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:51:37.741 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:37 smithi012 ceph-mon[26617]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 4.5 KiB/s rd, 195 KiB/s wr, 13 op/s 2024-04-26T04:51:37.742 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:37 smithi012 ceph-mon[37567]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 4.5 KiB/s rd, 195 KiB/s wr, 13 op/s 2024-04-26T04:51:37.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:37 smithi116 ceph-mon[29897]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 4.5 KiB/s rd, 195 KiB/s wr, 13 op/s 2024-04-26T04:51:39.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:39 smithi012 ceph-mon[26617]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 130 KiB/s wr, 9 op/s 2024-04-26T04:51:39.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:39 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T04:51:39.928 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:39 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:39.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:39 smithi012 ceph-mon[37567]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 130 KiB/s wr, 9 op/s 2024-04-26T04:51:39.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:39 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T04:51:39.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:39 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:39.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:39 smithi116 ceph-mon[29897]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 130 KiB/s wr, 9 op/s 2024-04-26T04:51:39.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:39 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T04:51:39.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:39 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:40.833 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:40 smithi012 ceph-mon[26617]: Deploying daemon osd.3 on smithi012 2024-04-26T04:51:40.833 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:40 smithi012 ceph-mon[37567]: Deploying daemon osd.3 on smithi012 2024-04-26T04:51:40.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:40 smithi116 ceph-mon[29897]: Deploying daemon osd.3 on smithi012 2024-04-26T04:51:41.909 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:41 smithi012 ceph-mon[26617]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-04-26T04:51:41.909 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:41 smithi012 ceph-mon[37567]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-04-26T04:51:41.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:41 smithi116 ceph-mon[29897]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-04-26T04:51:43.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:43 smithi012 ceph-mon[26617]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-26T04:51:43.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:43 smithi012 ceph-mon[37567]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-26T04:51:43.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:43 smithi116 ceph-mon[29897]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-26T04:51:45.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:45 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:45.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:45 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:45.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:45 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:45.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:45 smithi012 ceph-mon[26617]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-26T04:51:45.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:45 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:45.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:45 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:45.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:45 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:45.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:45 smithi012 ceph-mon[37567]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-26T04:51:45.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:45 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:45.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:45 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:45.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:45 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:45.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:45 smithi116 ceph-mon[29897]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-26T04:51:47.198 INFO:teuthology.orchestra.run.smithi012.stdout:Created osd(s) 3 on host 'smithi012' 2024-04-26T04:51:47.726 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:47 smithi012 ceph-mon[26617]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-26T04:51:47.726 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:47 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:47.726 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:47 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:47.727 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:47 smithi012 ceph-mon[26617]: from='osd.3 [v2:172.21.15.12:6826/3374180434,v1:172.21.15.12:6827/3374180434]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:51:47.727 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:47 smithi012 ceph-mon[26617]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:51:47.727 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:47 smithi012 ceph-mon[37567]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-26T04:51:47.727 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:47 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:47.727 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:47 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:47.727 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:47 smithi012 ceph-mon[37567]: from='osd.3 [v2:172.21.15.12:6826/3374180434,v1:172.21.15.12:6827/3374180434]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:51:47.727 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:47 smithi012 ceph-mon[37567]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:51:47.752 DEBUG:teuthology.orchestra.run.smithi012:osd.3> sudo journalctl -f -n 0 -u ceph-0b9350c6-0388-11ef-bc93-c7b262605968@osd.3.service 2024-04-26T04:51:47.754 INFO:tasks.cephadm:Deploying osd.4 on smithi116 with /dev/vg_nvme/lv_4... 2024-04-26T04:51:47.754 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-26T04:51:47.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:47 smithi116 ceph-mon[29897]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-26T04:51:47.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:47 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:47.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:47 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:47.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:47 smithi116 ceph-mon[29897]: from='osd.3 [v2:172.21.15.12:6826/3374180434,v1:172.21.15.12:6827/3374180434]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:51:47.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:47 smithi116 ceph-mon[29897]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:51:48.008 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:51:49.039 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:48 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:49.039 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:48 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:49.039 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:48 smithi012 ceph-mon[26617]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T04:51:49.040 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:48 smithi012 ceph-mon[26617]: osdmap e24: 4 total, 3 up, 4 in 2024-04-26T04:51:49.040 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:48 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:51:49.040 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:48 smithi012 ceph-mon[26617]: from='osd.3 [v2:172.21.15.12:6826/3374180434,v1:172.21.15.12:6827/3374180434]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-04-26T04:51:49.040 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:48 smithi012 ceph-mon[26617]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-04-26T04:51:49.040 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:48 smithi012 ceph-mon[26617]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:51:49.041 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:48 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:49.041 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:48 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:49.041 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:48 smithi012 ceph-mon[37567]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T04:51:49.041 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:48 smithi012 ceph-mon[37567]: osdmap e24: 4 total, 3 up, 4 in 2024-04-26T04:51:49.041 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:48 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:51:49.042 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:48 smithi012 ceph-mon[37567]: from='osd.3 [v2:172.21.15.12:6826/3374180434,v1:172.21.15.12:6827/3374180434]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-04-26T04:51:49.042 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:48 smithi012 ceph-mon[37567]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-04-26T04:51:49.042 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:48 smithi012 ceph-mon[37567]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:51:49.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:48 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:49.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:48 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:49.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:48 smithi116 ceph-mon[29897]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T04:51:49.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:48 smithi116 ceph-mon[29897]: osdmap e24: 4 total, 3 up, 4 in 2024-04-26T04:51:49.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:48 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:51:49.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:48 smithi116 ceph-mon[29897]: from='osd.3 [v2:172.21.15.12:6826/3374180434,v1:172.21.15.12:6827/3374180434]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-04-26T04:51:49.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:48 smithi116 ceph-mon[29897]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-04-26T04:51:49.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:48 smithi116 ceph-mon[29897]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:51:49.384 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 04:51:49 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3[58788]: 2024-04-26T04:51:49.197+0000 7f758584d700 -1 osd.3 0 waiting for initial osdmap 2024-04-26T04:51:49.384 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 04:51:49 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3[58788]: 2024-04-26T04:51:49.201+0000 7f757fe47700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:51:49.902 INFO:teuthology.orchestra.run.smithi116.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-26T04:51:49.902 INFO:teuthology.orchestra.run.smithi116.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-26T04:51:49.902 INFO:teuthology.orchestra.run.smithi116.stderr: stderr: 10+0 records in 2024-04-26T04:51:49.902 INFO:teuthology.orchestra.run.smithi116.stderr:10+0 records out 2024-04-26T04:51:49.902 INFO:teuthology.orchestra.run.smithi116.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0113918 s, 920 MB/s 2024-04-26T04:51:49.902 INFO:teuthology.orchestra.run.smithi116.stderr:--> Zapping successful for: 2024-04-26T04:51:50.448 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:50 smithi116 ceph-mon[29897]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2024-04-26T04:51:50.448 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:50 smithi116 ceph-mon[29897]: osdmap e25: 4 total, 3 up, 4 in 2024-04-26T04:51:50.448 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:50 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:51:50.448 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:50 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:51:50.473 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi116:vg_nvme/lv_4 2024-04-26T04:51:50.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:50 smithi012 ceph-mon[26617]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2024-04-26T04:51:50.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:50 smithi012 ceph-mon[26617]: osdmap e25: 4 total, 3 up, 4 in 2024-04-26T04:51:50.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:50 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:51:50.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:50 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:51:50.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:50 smithi012 ceph-mon[37567]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2024-04-26T04:51:50.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:50 smithi012 ceph-mon[37567]: osdmap e25: 4 total, 3 up, 4 in 2024-04-26T04:51:50.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:50 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:51:50.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:50 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:51:50.726 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:51:51.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:51 smithi012 ceph-mon[26617]: purged_snaps scrub starts 2024-04-26T04:51:51.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:51 smithi012 ceph-mon[26617]: purged_snaps scrub ok 2024-04-26T04:51:51.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:51 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:51:51.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:51 smithi012 ceph-mon[26617]: osd.3 [v2:172.21.15.12:6826/3374180434,v1:172.21.15.12:6827/3374180434] boot 2024-04-26T04:51:51.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:51 smithi012 ceph-mon[26617]: osdmap e26: 4 total, 4 up, 4 in 2024-04-26T04:51:51.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:51 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:51:51.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:51 smithi012 ceph-mon[26617]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:51:51.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:51 smithi012 ceph-mon[37567]: purged_snaps scrub starts 2024-04-26T04:51:51.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:51 smithi012 ceph-mon[37567]: purged_snaps scrub ok 2024-04-26T04:51:51.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:51 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:51:51.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:51 smithi012 ceph-mon[37567]: osd.3 [v2:172.21.15.12:6826/3374180434,v1:172.21.15.12:6827/3374180434] boot 2024-04-26T04:51:51.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:51 smithi012 ceph-mon[37567]: osdmap e26: 4 total, 4 up, 4 in 2024-04-26T04:51:51.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:51 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:51:51.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:51 smithi012 ceph-mon[37567]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:51:51.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:51 smithi116 ceph-mon[29897]: purged_snaps scrub starts 2024-04-26T04:51:51.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:51 smithi116 ceph-mon[29897]: purged_snaps scrub ok 2024-04-26T04:51:51.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:51 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:51:51.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:51 smithi116 ceph-mon[29897]: osd.3 [v2:172.21.15.12:6826/3374180434,v1:172.21.15.12:6827/3374180434] boot 2024-04-26T04:51:51.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:51 smithi116 ceph-mon[29897]: osdmap e26: 4 total, 4 up, 4 in 2024-04-26T04:51:51.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:51 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:51:51.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:51 smithi116 ceph-mon[29897]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:51:52.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:52 smithi116 ceph-mon[29897]: osdmap e27: 4 total, 4 up, 4 in 2024-04-26T04:51:52.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:52 smithi116 ceph-mon[29897]: Detected new or changed devices on smithi012 2024-04-26T04:51:52.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:52 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:52.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:52 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:52.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:52 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:51:52.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:52 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:51:52.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:52 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:51:52.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:52 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:51:52.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:52 smithi116 ceph-mon[29897]: Adjusting osd_memory_target on smithi012 to 4017M 2024-04-26T04:51:52.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:52 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:52.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:52 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:52.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:52 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:52.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:52 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:52.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:52 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:52.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:52 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:52.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:52 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:52.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:52 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:52.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:52 smithi012 ceph-mon[37567]: osdmap e27: 4 total, 4 up, 4 in 2024-04-26T04:51:52.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:52 smithi012 ceph-mon[37567]: Detected new or changed devices on smithi012 2024-04-26T04:51:52.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:52 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:52.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:52 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:52.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:52 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:51:52.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:52 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:51:52.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:52 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:51:52.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:52 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:51:52.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:52 smithi012 ceph-mon[37567]: Adjusting osd_memory_target on smithi012 to 4017M 2024-04-26T04:51:52.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:52 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:52.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:52 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:52.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:52 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:52.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:52 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:52.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:52 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:52.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:52 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:52.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:52 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:52.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:52 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:52.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:52 smithi012 ceph-mon[26617]: osdmap e27: 4 total, 4 up, 4 in 2024-04-26T04:51:52.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:52 smithi012 ceph-mon[26617]: Detected new or changed devices on smithi012 2024-04-26T04:51:52.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:52 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:52.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:52 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:52.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:52 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:51:52.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:52 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:51:52.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:52 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:51:52.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:52 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:51:52.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:52 smithi012 ceph-mon[26617]: Adjusting osd_memory_target on smithi012 to 4017M 2024-04-26T04:51:52.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:52 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:52.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:52 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:52.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:52 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:52.516 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:52 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:52.516 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:52 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:52.516 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:52 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:52.516 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:52 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:52.516 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:52 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:51:53.301 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:53 smithi116 ceph-mon[29897]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:51:53.302 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:53 smithi116 ceph-mon[29897]: from='client.24187 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi116:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:53.302 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:53 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:51:53.302 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:53 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:51:53.302 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:53 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:53.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:53 smithi012 ceph-mon[26617]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:51:53.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:53 smithi012 ceph-mon[26617]: from='client.24187 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi116:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:53.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:53 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:51:53.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:53 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:51:53.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:53 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:53.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:53 smithi012 ceph-mon[37567]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:51:53.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:53 smithi012 ceph-mon[37567]: from='client.24187 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi116:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:53.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:53 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:51:53.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:53 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:51:53.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:53 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:55.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:55 smithi116 ceph-mon[29897]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:51:55.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:55 smithi116 ceph-mon[29897]: from='client.? 172.21.15.116:0/3411687904' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ea9d3fb5-1877-4971-a8fb-8b5aafa7871a"}]: dispatch 2024-04-26T04:51:55.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:55 smithi116 ceph-mon[29897]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ea9d3fb5-1877-4971-a8fb-8b5aafa7871a"}]: dispatch 2024-04-26T04:51:55.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:55 smithi116 ceph-mon[29897]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ea9d3fb5-1877-4971-a8fb-8b5aafa7871a"}]': finished 2024-04-26T04:51:55.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:55 smithi116 ceph-mon[29897]: osdmap e28: 5 total, 4 up, 5 in 2024-04-26T04:51:55.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:55 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:51:55.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:55 smithi116 ceph-mon[29897]: from='client.? 172.21.15.116:0/2041326532' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:51:56.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:55 smithi012 ceph-mon[37567]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:51:56.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:55 smithi012 ceph-mon[37567]: from='client.? 172.21.15.116:0/3411687904' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ea9d3fb5-1877-4971-a8fb-8b5aafa7871a"}]: dispatch 2024-04-26T04:51:56.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:55 smithi012 ceph-mon[37567]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ea9d3fb5-1877-4971-a8fb-8b5aafa7871a"}]: dispatch 2024-04-26T04:51:56.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:55 smithi012 ceph-mon[37567]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ea9d3fb5-1877-4971-a8fb-8b5aafa7871a"}]': finished 2024-04-26T04:51:56.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:55 smithi012 ceph-mon[37567]: osdmap e28: 5 total, 4 up, 5 in 2024-04-26T04:51:56.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:55 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:51:56.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:55 smithi012 ceph-mon[37567]: from='client.? 172.21.15.116:0/2041326532' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:51:56.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:55 smithi012 ceph-mon[26617]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:51:56.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:55 smithi012 ceph-mon[26617]: from='client.? 172.21.15.116:0/3411687904' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ea9d3fb5-1877-4971-a8fb-8b5aafa7871a"}]: dispatch 2024-04-26T04:51:56.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:55 smithi012 ceph-mon[26617]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ea9d3fb5-1877-4971-a8fb-8b5aafa7871a"}]: dispatch 2024-04-26T04:51:56.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:55 smithi012 ceph-mon[26617]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ea9d3fb5-1877-4971-a8fb-8b5aafa7871a"}]': finished 2024-04-26T04:51:56.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:55 smithi012 ceph-mon[26617]: osdmap e28: 5 total, 4 up, 5 in 2024-04-26T04:51:56.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:55 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:51:56.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:55 smithi012 ceph-mon[26617]: from='client.? 172.21.15.116:0/2041326532' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:51:57.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:57 smithi116 ceph-mon[29897]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:51:58.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:57 smithi012 ceph-mon[26617]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:51:58.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:57 smithi012 ceph-mon[37567]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:51:59.847 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:51:59 smithi116 ceph-mon[29897]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:00.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:51:59 smithi012 ceph-mon[26617]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:00.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:51:59 smithi012 ceph-mon[37567]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:01.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:01 smithi116 ceph-mon[29897]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:01.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:01 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T04:52:01.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:01 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:01.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:01 smithi116 ceph-mon[29897]: Deploying daemon osd.4 on smithi116 2024-04-26T04:52:02.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:01 smithi012 ceph-mon[26617]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:02.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:01 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T04:52:02.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:01 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:02.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:01 smithi012 ceph-mon[26617]: Deploying daemon osd.4 on smithi116 2024-04-26T04:52:02.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:01 smithi012 ceph-mon[37567]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:02.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:01 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T04:52:02.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:01 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:02.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:01 smithi012 ceph-mon[37567]: Deploying daemon osd.4 on smithi116 2024-04-26T04:52:03.966 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:03 smithi116 ceph-mon[29897]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:04.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:03 smithi012 ceph-mon[26617]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:04.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:03 smithi012 ceph-mon[37567]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:05.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:05 smithi116 ceph-mon[29897]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:06.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:05 smithi012 ceph-mon[26617]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:06.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:05 smithi012 ceph-mon[37567]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:06.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:06 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:06.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:06 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:06.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:06 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:07.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:06 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:07.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:06 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:07.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:06 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:07.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:06 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:07.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:06 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:07.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:06 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:07.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:07 smithi116 ceph-mon[29897]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:08.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:07 smithi012 ceph-mon[26617]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:08.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:07 smithi012 ceph-mon[37567]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:08.595 INFO:teuthology.orchestra.run.smithi116.stdout:Created osd(s) 4 on host 'smithi116' 2024-04-26T04:52:09.442 DEBUG:teuthology.orchestra.run.smithi116:osd.4> sudo journalctl -f -n 0 -u ceph-0b9350c6-0388-11ef-bc93-c7b262605968@osd.4.service 2024-04-26T04:52:09.445 INFO:tasks.cephadm:Deploying osd.5 on smithi116 with /dev/vg_nvme/lv_3... 2024-04-26T04:52:09.445 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-26T04:52:09.715 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:09 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:09.715 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:09 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:09.715 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:09 smithi116 ceph-mon[29897]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:09.715 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:09 smithi116 ceph-mon[29897]: from='osd.4 [v2:172.21.15.116:6800/2011782592,v1:172.21.15.116:6801/2011782592]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T04:52:09.715 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:09 smithi116 ceph-mon[29897]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T04:52:09.746 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:52:10.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:09 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:10.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:09 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:10.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:09 smithi012 ceph-mon[26617]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:10.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:09 smithi012 ceph-mon[26617]: from='osd.4 [v2:172.21.15.116:6800/2011782592,v1:172.21.15.116:6801/2011782592]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T04:52:10.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:09 smithi012 ceph-mon[26617]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T04:52:10.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:09 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:10.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:09 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:10.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:09 smithi012 ceph-mon[37567]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:10.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:09 smithi012 ceph-mon[37567]: from='osd.4 [v2:172.21.15.116:6800/2011782592,v1:172.21.15.116:6801/2011782592]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T04:52:10.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:09 smithi012 ceph-mon[37567]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T04:52:10.826 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:10 smithi116 ceph-mon[29897]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-26T04:52:10.826 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:10 smithi116 ceph-mon[29897]: osdmap e29: 5 total, 4 up, 5 in 2024-04-26T04:52:10.826 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:10 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:52:10.827 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:10 smithi116 ceph-mon[29897]: from='osd.4 [v2:172.21.15.116:6800/2011782592,v1:172.21.15.116:6801/2011782592]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]: dispatch 2024-04-26T04:52:10.827 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:10 smithi116 ceph-mon[29897]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]: dispatch 2024-04-26T04:52:10.827 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:10 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:10.827 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:10 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:10.827 INFO:journalctl@ceph.osd.4.smithi116.stdout:Apr 26 04:52:10 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-4[34852]: 2024-04-26T04:52:10.564+0000 7f9a557a1700 -1 osd.4 0 waiting for initial osdmap 2024-04-26T04:52:10.827 INFO:journalctl@ceph.osd.4.smithi116.stdout:Apr 26 04:52:10 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-4[34852]: 2024-04-26T04:52:10.568+0000 7f9a525a0700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:52:10.914 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:10 smithi012 ceph-mon[37567]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-26T04:52:10.914 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:10 smithi012 ceph-mon[37567]: osdmap e29: 5 total, 4 up, 5 in 2024-04-26T04:52:10.915 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:10 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:52:10.915 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:10 smithi012 ceph-mon[37567]: from='osd.4 [v2:172.21.15.116:6800/2011782592,v1:172.21.15.116:6801/2011782592]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]: dispatch 2024-04-26T04:52:10.915 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:10 smithi012 ceph-mon[37567]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]: dispatch 2024-04-26T04:52:10.915 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:10 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:10.915 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:10 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:10.915 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:10 smithi012 ceph-mon[26617]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-26T04:52:10.915 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:10 smithi012 ceph-mon[26617]: osdmap e29: 5 total, 4 up, 5 in 2024-04-26T04:52:10.915 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:10 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:52:10.915 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:10 smithi012 ceph-mon[26617]: from='osd.4 [v2:172.21.15.116:6800/2011782592,v1:172.21.15.116:6801/2011782592]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]: dispatch 2024-04-26T04:52:10.915 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:10 smithi012 ceph-mon[26617]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]: dispatch 2024-04-26T04:52:10.915 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:10 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:10.915 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:10 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:11.662 INFO:teuthology.orchestra.run.smithi116.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-26T04:52:11.662 INFO:teuthology.orchestra.run.smithi116.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-26T04:52:11.662 INFO:teuthology.orchestra.run.smithi116.stderr: stderr: 10+0 records in 2024-04-26T04:52:11.662 INFO:teuthology.orchestra.run.smithi116.stderr:10+0 records out 2024-04-26T04:52:11.663 INFO:teuthology.orchestra.run.smithi116.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0161579 s, 649 MB/s 2024-04-26T04:52:11.663 INFO:teuthology.orchestra.run.smithi116.stderr: stderr: 2024-04-26T04:52:11.663 INFO:teuthology.orchestra.run.smithi116.stderr:--> Zapping successful for: 2024-04-26T04:52:11.727 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:11 smithi116 ceph-mon[29897]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]': finished 2024-04-26T04:52:11.727 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:11 smithi116 ceph-mon[29897]: osdmap e30: 5 total, 4 up, 5 in 2024-04-26T04:52:11.727 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:11 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:52:11.727 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:11 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:52:11.727 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:11 smithi116 ceph-mon[29897]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:11.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:11 smithi012 ceph-mon[26617]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]': finished 2024-04-26T04:52:11.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:11 smithi012 ceph-mon[26617]: osdmap e30: 5 total, 4 up, 5 in 2024-04-26T04:52:11.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:11 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:52:11.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:11 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:52:11.931 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:11 smithi012 ceph-mon[26617]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:11.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:11 smithi012 ceph-mon[37567]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]': finished 2024-04-26T04:52:11.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:11 smithi012 ceph-mon[37567]: osdmap e30: 5 total, 4 up, 5 in 2024-04-26T04:52:11.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:11 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:52:11.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:11 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:52:11.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:11 smithi012 ceph-mon[37567]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:52:12.483 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi116:vg_nvme/lv_3 2024-04-26T04:52:12.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:12 smithi116 ceph-mon[29897]: purged_snaps scrub starts 2024-04-26T04:52:12.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:12 smithi116 ceph-mon[29897]: purged_snaps scrub ok 2024-04-26T04:52:12.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:12 smithi116 ceph-mon[29897]: osd.4 [v2:172.21.15.116:6800/2011782592,v1:172.21.15.116:6801/2011782592] boot 2024-04-26T04:52:12.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:12 smithi116 ceph-mon[29897]: osdmap e31: 5 total, 5 up, 5 in 2024-04-26T04:52:12.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:12 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:52:12.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:12 smithi116 ceph-mon[29897]: osdmap e32: 5 total, 5 up, 5 in 2024-04-26T04:52:12.921 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:52:13.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:12 smithi012 ceph-mon[26617]: purged_snaps scrub starts 2024-04-26T04:52:13.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:12 smithi012 ceph-mon[26617]: purged_snaps scrub ok 2024-04-26T04:52:13.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:12 smithi012 ceph-mon[26617]: osd.4 [v2:172.21.15.116:6800/2011782592,v1:172.21.15.116:6801/2011782592] boot 2024-04-26T04:52:13.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:12 smithi012 ceph-mon[26617]: osdmap e31: 5 total, 5 up, 5 in 2024-04-26T04:52:13.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:12 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:52:13.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:12 smithi012 ceph-mon[26617]: osdmap e32: 5 total, 5 up, 5 in 2024-04-26T04:52:13.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:12 smithi012 ceph-mon[37567]: purged_snaps scrub starts 2024-04-26T04:52:13.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:12 smithi012 ceph-mon[37567]: purged_snaps scrub ok 2024-04-26T04:52:13.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:12 smithi012 ceph-mon[37567]: osd.4 [v2:172.21.15.116:6800/2011782592,v1:172.21.15.116:6801/2011782592] boot 2024-04-26T04:52:13.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:12 smithi012 ceph-mon[37567]: osdmap e31: 5 total, 5 up, 5 in 2024-04-26T04:52:13.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:12 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:52:13.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:12 smithi012 ceph-mon[37567]: osdmap e32: 5 total, 5 up, 5 in 2024-04-26T04:52:13.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:13 smithi116 ceph-mon[29897]: pgmap v110: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:52:13.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:13 smithi116 ceph-mon[29897]: osdmap e33: 5 total, 5 up, 5 in 2024-04-26T04:52:14.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:13 smithi012 ceph-mon[26617]: pgmap v110: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:52:14.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:13 smithi012 ceph-mon[26617]: osdmap e33: 5 total, 5 up, 5 in 2024-04-26T04:52:14.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:13 smithi012 ceph-mon[37567]: pgmap v110: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:52:14.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:13 smithi012 ceph-mon[37567]: osdmap e33: 5 total, 5 up, 5 in 2024-04-26T04:52:15.665 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:15 smithi116 ceph-mon[29897]: osdmap e34: 5 total, 5 up, 5 in 2024-04-26T04:52:15.665 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:15 smithi116 ceph-mon[29897]: pgmap v113: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:52:15.665 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:15 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:52:15.665 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:15 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:52:15.666 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:15 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:15.666 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:15 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:15.666 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:15 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:15.666 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:15 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:52:15.666 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:15 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:15.666 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:15 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:15.666 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:15 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:52:15.666 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:15 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:16.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:15 smithi012 ceph-mon[26617]: osdmap e34: 5 total, 5 up, 5 in 2024-04-26T04:52:16.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:15 smithi012 ceph-mon[26617]: pgmap v113: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:52:16.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:15 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:52:16.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:15 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:52:16.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:15 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:16.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:15 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:16.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:15 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:16.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:15 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:52:16.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:15 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:16.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:15 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:16.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:15 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:52:16.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:15 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:16.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:15 smithi012 ceph-mon[37567]: osdmap e34: 5 total, 5 up, 5 in 2024-04-26T04:52:16.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:15 smithi012 ceph-mon[37567]: pgmap v113: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:52:16.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:15 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:52:16.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:15 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:52:16.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:15 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:16.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:15 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:16.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:15 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:16.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:15 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:52:16.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:15 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:16.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:15 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:16.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:15 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:52:16.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:15 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:16.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:16 smithi116 ceph-mon[29897]: from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi116:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:52:16.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:16 smithi116 ceph-mon[29897]: Detected new or changed devices on smithi116 2024-04-26T04:52:16.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:16 smithi116 ceph-mon[29897]: Adjusting osd_memory_target on smithi116 to 17088M 2024-04-26T04:52:16.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:16 smithi116 ceph-mon[29897]: osdmap e35: 5 total, 5 up, 5 in 2024-04-26T04:52:16.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:16 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:16.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:16 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:16.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:16 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:52:16.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:16 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:17.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:16 smithi012 ceph-mon[26617]: from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi116:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:52:17.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:16 smithi012 ceph-mon[26617]: Detected new or changed devices on smithi116 2024-04-26T04:52:17.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:16 smithi012 ceph-mon[26617]: Adjusting osd_memory_target on smithi116 to 17088M 2024-04-26T04:52:17.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:16 smithi012 ceph-mon[26617]: osdmap e35: 5 total, 5 up, 5 in 2024-04-26T04:52:17.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:16 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:17.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:16 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:17.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:16 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:52:17.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:16 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:17.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:16 smithi012 ceph-mon[37567]: from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi116:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:52:17.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:16 smithi012 ceph-mon[37567]: Detected new or changed devices on smithi116 2024-04-26T04:52:17.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:16 smithi012 ceph-mon[37567]: Adjusting osd_memory_target on smithi116 to 17088M 2024-04-26T04:52:17.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:16 smithi012 ceph-mon[37567]: osdmap e35: 5 total, 5 up, 5 in 2024-04-26T04:52:17.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:16 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:17.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:16 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:17.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:16 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:52:17.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:16 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:17.899 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:17 smithi012 ceph-mon[26617]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 143 KiB/s, 0 objects/s recovering 2024-04-26T04:52:17.900 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:17 smithi012 ceph-mon[37567]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 143 KiB/s, 0 objects/s recovering 2024-04-26T04:52:17.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:17 smithi116 ceph-mon[29897]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 143 KiB/s, 0 objects/s recovering 2024-04-26T04:52:18.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:18 smithi116 ceph-mon[29897]: from='client.? 172.21.15.116:0/3560058157' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "59f0530d-37eb-41d8-86d0-580ba9b8e4e2"}]: dispatch 2024-04-26T04:52:18.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:18 smithi116 ceph-mon[29897]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "59f0530d-37eb-41d8-86d0-580ba9b8e4e2"}]: dispatch 2024-04-26T04:52:18.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:18 smithi116 ceph-mon[29897]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "59f0530d-37eb-41d8-86d0-580ba9b8e4e2"}]': finished 2024-04-26T04:52:18.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:18 smithi116 ceph-mon[29897]: osdmap e36: 6 total, 5 up, 6 in 2024-04-26T04:52:18.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:18 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:52:18.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:18 smithi116 ceph-mon[29897]: from='client.? 172.21.15.116:0/1565438808' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:52:19.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:18 smithi012 ceph-mon[26617]: from='client.? 172.21.15.116:0/3560058157' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "59f0530d-37eb-41d8-86d0-580ba9b8e4e2"}]: dispatch 2024-04-26T04:52:19.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:18 smithi012 ceph-mon[26617]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "59f0530d-37eb-41d8-86d0-580ba9b8e4e2"}]: dispatch 2024-04-26T04:52:19.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:18 smithi012 ceph-mon[26617]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "59f0530d-37eb-41d8-86d0-580ba9b8e4e2"}]': finished 2024-04-26T04:52:19.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:18 smithi012 ceph-mon[26617]: osdmap e36: 6 total, 5 up, 6 in 2024-04-26T04:52:19.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:18 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:52:19.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:18 smithi012 ceph-mon[26617]: from='client.? 172.21.15.116:0/1565438808' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:52:19.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:18 smithi012 ceph-mon[37567]: from='client.? 172.21.15.116:0/3560058157' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "59f0530d-37eb-41d8-86d0-580ba9b8e4e2"}]: dispatch 2024-04-26T04:52:19.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:18 smithi012 ceph-mon[37567]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "59f0530d-37eb-41d8-86d0-580ba9b8e4e2"}]: dispatch 2024-04-26T04:52:19.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:18 smithi012 ceph-mon[37567]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "59f0530d-37eb-41d8-86d0-580ba9b8e4e2"}]': finished 2024-04-26T04:52:19.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:18 smithi012 ceph-mon[37567]: osdmap e36: 6 total, 5 up, 6 in 2024-04-26T04:52:19.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:18 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:52:19.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:18 smithi012 ceph-mon[37567]: from='client.? 172.21.15.116:0/1565438808' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:52:19.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:19 smithi116 ceph-mon[29897]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 115 KiB/s, 0 objects/s recovering 2024-04-26T04:52:20.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:19 smithi012 ceph-mon[26617]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 115 KiB/s, 0 objects/s recovering 2024-04-26T04:52:20.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:19 smithi012 ceph-mon[37567]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 115 KiB/s, 0 objects/s recovering 2024-04-26T04:52:21.907 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:21 smithi116 ceph-mon[29897]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-26T04:52:22.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:21 smithi012 ceph-mon[26617]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-26T04:52:22.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:21 smithi012 ceph-mon[37567]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-26T04:52:23.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:23 smithi116 ceph-mon[29897]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-26T04:52:24.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:23 smithi012 ceph-mon[26617]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-26T04:52:24.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:23 smithi012 ceph-mon[37567]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-26T04:52:24.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:24 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T04:52:24.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:24 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:24.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:24 smithi116 ceph-mon[29897]: Deploying daemon osd.5 on smithi116 2024-04-26T04:52:25.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:24 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T04:52:25.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:24 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:25.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:24 smithi012 ceph-mon[26617]: Deploying daemon osd.5 on smithi116 2024-04-26T04:52:25.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:24 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T04:52:25.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:24 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:25.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:24 smithi012 ceph-mon[37567]: Deploying daemon osd.5 on smithi116 2024-04-26T04:52:25.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:25 smithi116 ceph-mon[29897]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-04-26T04:52:26.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:25 smithi012 ceph-mon[26617]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-04-26T04:52:26.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:25 smithi012 ceph-mon[37567]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-04-26T04:52:27.862 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:27 smithi116 ceph-mon[29897]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:52:28.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:27 smithi012 ceph-mon[26617]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:52:28.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:27 smithi012 ceph-mon[37567]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:52:29.847 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:29 smithi116 ceph-mon[29897]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:52:29.847 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:29 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:29.848 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:29 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:29.848 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:29 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:30.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:29 smithi012 ceph-mon[26617]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:52:30.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:29 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:30.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:29 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:30.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:29 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:30.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:29 smithi012 ceph-mon[37567]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:52:30.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:29 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:30.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:29 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:30.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:29 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:31.966 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:31 smithi116 ceph-mon[29897]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:52:31.967 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:31 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:31.967 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:31 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:31.967 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:31 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:31.967 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:31 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:52:31.967 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:31 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:32.035 INFO:teuthology.orchestra.run.smithi116.stdout:Created osd(s) 5 on host 'smithi116' 2024-04-26T04:52:32.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:31 smithi012 ceph-mon[26617]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:52:32.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:31 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:32.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:31 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:32.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:31 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:32.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:31 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:52:32.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:31 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:32.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:31 smithi012 ceph-mon[37567]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:52:32.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:31 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:32.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:31 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:32.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:31 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:32.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:31 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:52:32.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:31 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:32.607 DEBUG:teuthology.orchestra.run.smithi116:osd.5> sudo journalctl -f -n 0 -u ceph-0b9350c6-0388-11ef-bc93-c7b262605968@osd.5.service 2024-04-26T04:52:32.609 INFO:tasks.cephadm:Deploying osd.6 on smithi116 with /dev/vg_nvme/lv_2... 2024-04-26T04:52:32.610 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-26T04:52:32.902 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:32 smithi116 ceph-mon[29897]: from='osd.5 [v2:172.21.15.116:6808/4077435109,v1:172.21.15.116:6809/4077435109]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T04:52:32.902 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:32 smithi116 ceph-mon[29897]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T04:52:32.902 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:32 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:32.902 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:32 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:32.902 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:32 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:33.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:32 smithi012 ceph-mon[26617]: from='osd.5 [v2:172.21.15.116:6808/4077435109,v1:172.21.15.116:6809/4077435109]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T04:52:33.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:32 smithi012 ceph-mon[26617]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T04:52:33.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:32 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:33.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:32 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:33.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:32 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:33.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:32 smithi012 ceph-mon[37567]: from='osd.5 [v2:172.21.15.116:6808/4077435109,v1:172.21.15.116:6809/4077435109]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T04:52:33.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:32 smithi012 ceph-mon[37567]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T04:52:33.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:32 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:33.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:32 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:33.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:32 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:33.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:33 smithi116 ceph-mon[29897]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:52:33.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:33 smithi116 ceph-mon[29897]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-26T04:52:33.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:33 smithi116 ceph-mon[29897]: osdmap e37: 6 total, 5 up, 6 in 2024-04-26T04:52:33.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:33 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:52:33.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:33 smithi116 ceph-mon[29897]: from='osd.5 [v2:172.21.15.116:6808/4077435109,v1:172.21.15.116:6809/4077435109]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]: dispatch 2024-04-26T04:52:33.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:33 smithi116 ceph-mon[29897]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]: dispatch 2024-04-26T04:52:33.992 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 04:52:33 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T04:52:33.784+0000 7fcc8cd87700 -1 osd.5 0 waiting for initial osdmap 2024-04-26T04:52:33.993 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 04:52:33 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T04:52:33.791+0000 7fcc87381700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:52:34.025 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:52:34.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:33 smithi012 ceph-mon[26617]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:52:34.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:33 smithi012 ceph-mon[26617]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-26T04:52:34.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:33 smithi012 ceph-mon[26617]: osdmap e37: 6 total, 5 up, 6 in 2024-04-26T04:52:34.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:33 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:52:34.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:33 smithi012 ceph-mon[26617]: from='osd.5 [v2:172.21.15.116:6808/4077435109,v1:172.21.15.116:6809/4077435109]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]: dispatch 2024-04-26T04:52:34.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:33 smithi012 ceph-mon[26617]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]: dispatch 2024-04-26T04:52:34.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:33 smithi012 ceph-mon[37567]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:52:34.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:33 smithi012 ceph-mon[37567]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-26T04:52:34.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:33 smithi012 ceph-mon[37567]: osdmap e37: 6 total, 5 up, 6 in 2024-04-26T04:52:34.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:33 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:52:34.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:33 smithi012 ceph-mon[37567]: from='osd.5 [v2:172.21.15.116:6808/4077435109,v1:172.21.15.116:6809/4077435109]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]: dispatch 2024-04-26T04:52:34.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:33 smithi012 ceph-mon[37567]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]: dispatch 2024-04-26T04:52:34.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:34 smithi116 ceph-mon[29897]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]': finished 2024-04-26T04:52:34.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:34 smithi116 ceph-mon[29897]: osdmap e38: 6 total, 5 up, 6 in 2024-04-26T04:52:34.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:34 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:52:34.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:34 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:52:35.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:34 smithi012 ceph-mon[26617]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]': finished 2024-04-26T04:52:35.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:34 smithi012 ceph-mon[26617]: osdmap e38: 6 total, 5 up, 6 in 2024-04-26T04:52:35.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:34 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:52:35.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:34 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:52:35.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:34 smithi012 ceph-mon[37567]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]': finished 2024-04-26T04:52:35.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:34 smithi012 ceph-mon[37567]: osdmap e38: 6 total, 5 up, 6 in 2024-04-26T04:52:35.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:34 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:52:35.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:34 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:52:36.106 INFO:teuthology.orchestra.run.smithi116.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-26T04:52:36.106 INFO:teuthology.orchestra.run.smithi116.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-26T04:52:36.107 INFO:teuthology.orchestra.run.smithi116.stderr: stderr: 10+0 records in 2024-04-26T04:52:36.107 INFO:teuthology.orchestra.run.smithi116.stderr:10+0 records out 2024-04-26T04:52:36.107 INFO:teuthology.orchestra.run.smithi116.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0128266 s, 817 MB/s 2024-04-26T04:52:36.107 INFO:teuthology.orchestra.run.smithi116.stderr:--> Zapping successful for: 2024-04-26T04:52:36.107 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:35 smithi116 ceph-mon[29897]: purged_snaps scrub starts 2024-04-26T04:52:36.107 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:35 smithi116 ceph-mon[29897]: purged_snaps scrub ok 2024-04-26T04:52:36.107 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:35 smithi116 ceph-mon[29897]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:52:36.107 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:35 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:52:36.107 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:35 smithi116 ceph-mon[29897]: osd.5 [v2:172.21.15.116:6808/4077435109,v1:172.21.15.116:6809/4077435109] boot 2024-04-26T04:52:36.107 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:35 smithi116 ceph-mon[29897]: osdmap e39: 6 total, 6 up, 6 in 2024-04-26T04:52:36.107 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:35 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:52:36.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:35 smithi012 ceph-mon[26617]: purged_snaps scrub starts 2024-04-26T04:52:36.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:35 smithi012 ceph-mon[26617]: purged_snaps scrub ok 2024-04-26T04:52:36.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:35 smithi012 ceph-mon[26617]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:52:36.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:35 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:52:36.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:35 smithi012 ceph-mon[26617]: osd.5 [v2:172.21.15.116:6808/4077435109,v1:172.21.15.116:6809/4077435109] boot 2024-04-26T04:52:36.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:35 smithi012 ceph-mon[26617]: osdmap e39: 6 total, 6 up, 6 in 2024-04-26T04:52:36.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:35 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:52:36.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:35 smithi012 ceph-mon[37567]: purged_snaps scrub starts 2024-04-26T04:52:36.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:35 smithi012 ceph-mon[37567]: purged_snaps scrub ok 2024-04-26T04:52:36.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:35 smithi012 ceph-mon[37567]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:52:36.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:35 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:52:36.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:35 smithi012 ceph-mon[37567]: osd.5 [v2:172.21.15.116:6808/4077435109,v1:172.21.15.116:6809/4077435109] boot 2024-04-26T04:52:36.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:35 smithi012 ceph-mon[37567]: osdmap e39: 6 total, 6 up, 6 in 2024-04-26T04:52:36.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:35 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:52:36.815 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi116:vg_nvme/lv_2 2024-04-26T04:52:37.061 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:52:37.184 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:36 smithi116 ceph-mon[29897]: osdmap e40: 6 total, 6 up, 6 in 2024-04-26T04:52:37.184 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:36 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:37.184 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:36 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:37.184 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:36 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:52:37.184 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:36 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:52:37.184 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:36 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:37.184 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:36 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:37.184 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:36 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:52:37.185 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:36 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:37.185 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:36 smithi116 ceph-mon[29897]: osdmap e41: 6 total, 6 up, 6 in 2024-04-26T04:52:37.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:36 smithi012 ceph-mon[26617]: osdmap e40: 6 total, 6 up, 6 in 2024-04-26T04:52:37.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:36 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:37.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:36 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:37.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:36 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:52:37.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:36 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:52:37.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:36 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:37.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:36 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:37.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:36 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:52:37.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:36 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:37.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:36 smithi012 ceph-mon[26617]: osdmap e41: 6 total, 6 up, 6 in 2024-04-26T04:52:37.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:36 smithi012 ceph-mon[37567]: osdmap e40: 6 total, 6 up, 6 in 2024-04-26T04:52:37.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:36 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:37.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:36 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:37.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:36 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:52:37.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:36 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:52:37.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:36 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:37.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:36 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:37.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:36 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:52:37.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:36 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:37.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:36 smithi012 ceph-mon[37567]: osdmap e41: 6 total, 6 up, 6 in 2024-04-26T04:52:38.124 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:37 smithi116 ceph-mon[29897]: Detected new or changed devices on smithi116 2024-04-26T04:52:38.124 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:37 smithi116 ceph-mon[29897]: Adjusting osd_memory_target on smithi116 to 8544M 2024-04-26T04:52:38.124 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:37 smithi116 ceph-mon[29897]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:52:38.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:37 smithi012 ceph-mon[26617]: Detected new or changed devices on smithi116 2024-04-26T04:52:38.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:37 smithi012 ceph-mon[26617]: Adjusting osd_memory_target on smithi116 to 8544M 2024-04-26T04:52:38.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:37 smithi012 ceph-mon[26617]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:52:38.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:37 smithi012 ceph-mon[37567]: Detected new or changed devices on smithi116 2024-04-26T04:52:38.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:37 smithi012 ceph-mon[37567]: Adjusting osd_memory_target on smithi116 to 8544M 2024-04-26T04:52:38.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:37 smithi012 ceph-mon[37567]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:52:39.120 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:38 smithi116 ceph-mon[29897]: osdmap e42: 6 total, 6 up, 6 in 2024-04-26T04:52:39.120 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:38 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:52:39.120 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:38 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:52:39.120 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:38 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:39.121 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:38 smithi116 ceph-mon[29897]: osdmap e43: 6 total, 6 up, 6 in 2024-04-26T04:52:39.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:38 smithi012 ceph-mon[26617]: osdmap e42: 6 total, 6 up, 6 in 2024-04-26T04:52:39.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:38 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:52:39.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:38 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:52:39.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:38 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:39.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:38 smithi012 ceph-mon[26617]: osdmap e43: 6 total, 6 up, 6 in 2024-04-26T04:52:39.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:38 smithi012 ceph-mon[37567]: osdmap e42: 6 total, 6 up, 6 in 2024-04-26T04:52:39.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:38 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:52:39.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:38 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:52:39.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:38 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:39.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:38 smithi012 ceph-mon[37567]: osdmap e43: 6 total, 6 up, 6 in 2024-04-26T04:52:39.963 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:39 smithi116 ceph-mon[29897]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:52:39.963 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:39 smithi116 ceph-mon[29897]: from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi116:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:52:40.164 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:39 smithi012 ceph-mon[26617]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:52:40.164 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:39 smithi012 ceph-mon[26617]: from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi116:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:52:40.164 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:39 smithi012 ceph-mon[37567]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:52:40.165 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:39 smithi012 ceph-mon[37567]: from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi116:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:52:42.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:41 smithi116 ceph-mon[29897]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:52:42.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:41 smithi116 ceph-mon[29897]: from='client.? 172.21.15.116:0/1709392784' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "62b1edf5-75b9-4ea6-af34-d40584a6c343"}]: dispatch 2024-04-26T04:52:42.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:41 smithi116 ceph-mon[29897]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "62b1edf5-75b9-4ea6-af34-d40584a6c343"}]: dispatch 2024-04-26T04:52:42.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:41 smithi116 ceph-mon[29897]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "62b1edf5-75b9-4ea6-af34-d40584a6c343"}]': finished 2024-04-26T04:52:42.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:41 smithi116 ceph-mon[29897]: osdmap e44: 7 total, 6 up, 7 in 2024-04-26T04:52:42.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:41 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:52:42.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:41 smithi116 ceph-mon[29897]: from='client.? 172.21.15.116:0/3909941504' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:52:42.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:41 smithi012 ceph-mon[26617]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:52:42.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:41 smithi012 ceph-mon[26617]: from='client.? 172.21.15.116:0/1709392784' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "62b1edf5-75b9-4ea6-af34-d40584a6c343"}]: dispatch 2024-04-26T04:52:42.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:41 smithi012 ceph-mon[26617]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "62b1edf5-75b9-4ea6-af34-d40584a6c343"}]: dispatch 2024-04-26T04:52:42.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:41 smithi012 ceph-mon[26617]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "62b1edf5-75b9-4ea6-af34-d40584a6c343"}]': finished 2024-04-26T04:52:42.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:41 smithi012 ceph-mon[26617]: osdmap e44: 7 total, 6 up, 7 in 2024-04-26T04:52:42.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:41 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:52:42.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:41 smithi012 ceph-mon[26617]: from='client.? 172.21.15.116:0/3909941504' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:52:42.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:41 smithi012 ceph-mon[37567]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:52:42.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:41 smithi012 ceph-mon[37567]: from='client.? 172.21.15.116:0/1709392784' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "62b1edf5-75b9-4ea6-af34-d40584a6c343"}]: dispatch 2024-04-26T04:52:42.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:41 smithi012 ceph-mon[37567]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "62b1edf5-75b9-4ea6-af34-d40584a6c343"}]: dispatch 2024-04-26T04:52:42.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:41 smithi012 ceph-mon[37567]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "62b1edf5-75b9-4ea6-af34-d40584a6c343"}]': finished 2024-04-26T04:52:42.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:41 smithi012 ceph-mon[37567]: osdmap e44: 7 total, 6 up, 7 in 2024-04-26T04:52:42.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:41 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:52:42.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:41 smithi012 ceph-mon[37567]: from='client.? 172.21.15.116:0/3909941504' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:52:44.153 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:43 smithi116 ceph-mon[29897]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:52:44.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:43 smithi012 ceph-mon[26617]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:52:44.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:43 smithi012 ceph-mon[37567]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:52:46.134 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:45 smithi116 ceph-mon[29897]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:52:46.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:45 smithi012 ceph-mon[26617]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:52:46.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:45 smithi012 ceph-mon[37567]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:52:48.123 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:47 smithi116 ceph-mon[29897]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:52:48.123 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:47 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T04:52:48.123 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:47 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:48.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:47 smithi012 ceph-mon[26617]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:52:48.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:47 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T04:52:48.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:47 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:48.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:47 smithi012 ceph-mon[37567]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:52:48.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:47 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T04:52:48.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:47 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:49.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:48 smithi116 ceph-mon[29897]: Deploying daemon osd.6 on smithi116 2024-04-26T04:52:49.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:48 smithi012 ceph-mon[26617]: Deploying daemon osd.6 on smithi116 2024-04-26T04:52:49.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:48 smithi012 ceph-mon[37567]: Deploying daemon osd.6 on smithi116 2024-04-26T04:52:49.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:49 smithi116 ceph-mon[29897]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:52:50.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:49 smithi012 ceph-mon[26617]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:52:50.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:49 smithi012 ceph-mon[37567]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:52:51.948 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:51 smithi116 ceph-mon[29897]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:52:52.262 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:51 smithi012 ceph-mon[37567]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:52:52.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:51 smithi012 ceph-mon[26617]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:52:53.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:53 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:53.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:53 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:53.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:53 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:53.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:53 smithi116 ceph-mon[29897]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:52:53.512 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:53 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:53.512 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:53 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:53.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:53 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:53.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:53 smithi012 ceph-mon[26617]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:52:53.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:53 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:53.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:53 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:53.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:53 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:53.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:53 smithi012 ceph-mon[37567]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:52:55.393 INFO:teuthology.orchestra.run.smithi116.stdout:Created osd(s) 6 on host 'smithi116' 2024-04-26T04:52:55.978 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:55 smithi116 ceph-mon[29897]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:52:55.978 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:55 smithi116 ceph-mon[29897]: from='osd.6 [v2:172.21.15.116:6816/2560073849,v1:172.21.15.116:6817/2560073849]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T04:52:55.978 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:55 smithi116 ceph-mon[29897]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T04:52:55.978 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:55 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:55.979 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:55 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:56.002 DEBUG:teuthology.orchestra.run.smithi116:osd.6> sudo journalctl -f -n 0 -u ceph-0b9350c6-0388-11ef-bc93-c7b262605968@osd.6.service 2024-04-26T04:52:56.004 INFO:tasks.cephadm:Deploying osd.7 on smithi116 with /dev/vg_nvme/lv_1... 2024-04-26T04:52:56.005 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-26T04:52:56.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:55 smithi012 ceph-mon[26617]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:52:56.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:55 smithi012 ceph-mon[26617]: from='osd.6 [v2:172.21.15.116:6816/2560073849,v1:172.21.15.116:6817/2560073849]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T04:52:56.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:55 smithi012 ceph-mon[26617]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T04:52:56.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:55 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:56.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:55 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:56.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:55 smithi012 ceph-mon[37567]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:52:56.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:55 smithi012 ceph-mon[37567]: from='osd.6 [v2:172.21.15.116:6816/2560073849,v1:172.21.15.116:6817/2560073849]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T04:52:56.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:55 smithi012 ceph-mon[37567]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T04:52:56.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:55 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:56.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:55 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:56.313 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:52:56.699 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:56 smithi116 ceph-mon[29897]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-26T04:52:56.699 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:56 smithi116 ceph-mon[29897]: osdmap e45: 7 total, 6 up, 7 in 2024-04-26T04:52:56.699 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:56 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:52:56.699 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:56 smithi116 ceph-mon[29897]: from='osd.6 [v2:172.21.15.116:6816/2560073849,v1:172.21.15.116:6817/2560073849]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]: dispatch 2024-04-26T04:52:56.699 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:56 smithi116 ceph-mon[29897]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]: dispatch 2024-04-26T04:52:56.699 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:56 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:56.699 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:56 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:56.699 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 04:52:56 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T04:52:56.674+0000 7fe122c28700 -1 osd.6 0 waiting for initial osdmap 2024-04-26T04:52:56.699 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 04:52:56 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T04:52:56.680+0000 7fe11a21c700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:52:57.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:56 smithi012 ceph-mon[26617]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-26T04:52:57.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:56 smithi012 ceph-mon[26617]: osdmap e45: 7 total, 6 up, 7 in 2024-04-26T04:52:57.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:56 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:52:57.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:56 smithi012 ceph-mon[26617]: from='osd.6 [v2:172.21.15.116:6816/2560073849,v1:172.21.15.116:6817/2560073849]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]: dispatch 2024-04-26T04:52:57.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:56 smithi012 ceph-mon[26617]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]: dispatch 2024-04-26T04:52:57.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:56 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:57.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:56 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:57.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:56 smithi012 ceph-mon[37567]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-26T04:52:57.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:56 smithi012 ceph-mon[37567]: osdmap e45: 7 total, 6 up, 7 in 2024-04-26T04:52:57.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:56 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:52:57.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:56 smithi012 ceph-mon[37567]: from='osd.6 [v2:172.21.15.116:6816/2560073849,v1:172.21.15.116:6817/2560073849]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]: dispatch 2024-04-26T04:52:57.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:56 smithi012 ceph-mon[37567]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]: dispatch 2024-04-26T04:52:57.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:56 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:57.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:56 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:52:57.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:57 smithi116 ceph-mon[29897]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:52:57.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:57 smithi116 ceph-mon[29897]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]': finished 2024-04-26T04:52:57.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:57 smithi116 ceph-mon[29897]: osdmap e46: 7 total, 6 up, 7 in 2024-04-26T04:52:57.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:57 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:52:57.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:57 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:52:58.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:57 smithi012 ceph-mon[26617]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:52:58.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:57 smithi012 ceph-mon[26617]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]': finished 2024-04-26T04:52:58.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:57 smithi012 ceph-mon[26617]: osdmap e46: 7 total, 6 up, 7 in 2024-04-26T04:52:58.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:57 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:52:58.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:57 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:52:58.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:57 smithi012 ceph-mon[37567]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:52:58.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:57 smithi012 ceph-mon[37567]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]': finished 2024-04-26T04:52:58.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:57 smithi012 ceph-mon[37567]: osdmap e46: 7 total, 6 up, 7 in 2024-04-26T04:52:58.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:57 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:52:58.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:57 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:52:58.224 INFO:teuthology.orchestra.run.smithi116.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-26T04:52:58.224 INFO:teuthology.orchestra.run.smithi116.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-26T04:52:58.224 INFO:teuthology.orchestra.run.smithi116.stderr: stderr: 10+0 records in 2024-04-26T04:52:58.224 INFO:teuthology.orchestra.run.smithi116.stderr:10+0 records out 2024-04-26T04:52:58.224 INFO:teuthology.orchestra.run.smithi116.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0153667 s, 682 MB/s 2024-04-26T04:52:58.224 INFO:teuthology.orchestra.run.smithi116.stderr:--> Zapping successful for: 2024-04-26T04:52:58.937 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:58 smithi116 ceph-mon[29897]: purged_snaps scrub starts 2024-04-26T04:52:58.937 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:58 smithi116 ceph-mon[29897]: purged_snaps scrub ok 2024-04-26T04:52:58.937 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:58 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:52:58.937 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:58 smithi116 ceph-mon[29897]: osd.6 [v2:172.21.15.116:6816/2560073849,v1:172.21.15.116:6817/2560073849] boot 2024-04-26T04:52:58.937 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:58 smithi116 ceph-mon[29897]: osdmap e47: 7 total, 7 up, 7 in 2024-04-26T04:52:58.937 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:58 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:52:58.937 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:58 smithi116 ceph-mon[29897]: osdmap e48: 7 total, 7 up, 7 in 2024-04-26T04:52:59.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:58 smithi012 ceph-mon[26617]: purged_snaps scrub starts 2024-04-26T04:52:59.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:58 smithi012 ceph-mon[26617]: purged_snaps scrub ok 2024-04-26T04:52:59.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:58 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:52:59.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:58 smithi012 ceph-mon[26617]: osd.6 [v2:172.21.15.116:6816/2560073849,v1:172.21.15.116:6817/2560073849] boot 2024-04-26T04:52:59.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:58 smithi012 ceph-mon[26617]: osdmap e47: 7 total, 7 up, 7 in 2024-04-26T04:52:59.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:58 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:52:59.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:58 smithi012 ceph-mon[26617]: osdmap e48: 7 total, 7 up, 7 in 2024-04-26T04:52:59.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:58 smithi012 ceph-mon[37567]: purged_snaps scrub starts 2024-04-26T04:52:59.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:58 smithi012 ceph-mon[37567]: purged_snaps scrub ok 2024-04-26T04:52:59.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:58 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:52:59.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:58 smithi012 ceph-mon[37567]: osd.6 [v2:172.21.15.116:6816/2560073849,v1:172.21.15.116:6817/2560073849] boot 2024-04-26T04:52:59.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:58 smithi012 ceph-mon[37567]: osdmap e47: 7 total, 7 up, 7 in 2024-04-26T04:52:59.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:58 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:52:59.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:58 smithi012 ceph-mon[37567]: osdmap e48: 7 total, 7 up, 7 in 2024-04-26T04:52:59.216 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi116:vg_nvme/lv_1 2024-04-26T04:52:59.566 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:52:59.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:59 smithi116 ceph-mon[29897]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:52:59.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:52:59 smithi116 ceph-mon[29897]: osdmap e49: 7 total, 7 up, 7 in 2024-04-26T04:53:00.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:59 smithi012 ceph-mon[26617]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:53:00.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:52:59 smithi012 ceph-mon[26617]: osdmap e49: 7 total, 7 up, 7 in 2024-04-26T04:53:00.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:59 smithi012 ceph-mon[37567]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:53:00.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:52:59 smithi012 ceph-mon[37567]: osdmap e49: 7 total, 7 up, 7 in 2024-04-26T04:53:01.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:00 smithi116 ceph-mon[29897]: osdmap e50: 7 total, 7 up, 7 in 2024-04-26T04:53:01.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:00 smithi116 ceph-mon[29897]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:53:01.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:00 smithi012 ceph-mon[26617]: osdmap e50: 7 total, 7 up, 7 in 2024-04-26T04:53:01.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:00 smithi012 ceph-mon[26617]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:53:01.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:00 smithi012 ceph-mon[37567]: osdmap e50: 7 total, 7 up, 7 in 2024-04-26T04:53:01.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:00 smithi012 ceph-mon[37567]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:53:02.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:01 smithi116 ceph-mon[29897]: osdmap e51: 7 total, 7 up, 7 in 2024-04-26T04:53:02.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:01 smithi012 ceph-mon[26617]: osdmap e51: 7 total, 7 up, 7 in 2024-04-26T04:53:02.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:01 smithi012 ceph-mon[37567]: osdmap e51: 7 total, 7 up, 7 in 2024-04-26T04:53:03.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:03 smithi116 ceph-mon[29897]: Detected new or changed devices on smithi116 2024-04-26T04:53:03.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:03 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:03.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:03 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:03.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:03 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:03.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:03 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:03.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:03 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:03.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:03 smithi116 ceph-mon[29897]: Adjusting osd_memory_target on smithi116 to 5696M 2024-04-26T04:53:03.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:03 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:03.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:03 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:03.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:03 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:03.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:03 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:03.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:03 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:03.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:03 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:03.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:03 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:03.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:03 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:03.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:03 smithi116 ceph-mon[29897]: from='client.24268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi116:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:53:03.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:03 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:53:03.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:03 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:53:03.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:03 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:03.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:03 smithi116 ceph-mon[29897]: pgmap v154: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:53:03.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[26617]: Detected new or changed devices on smithi116 2024-04-26T04:53:03.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:03.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:03.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:03.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:03.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:03.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[26617]: Adjusting osd_memory_target on smithi116 to 5696M 2024-04-26T04:53:03.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:03.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:03.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:03.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:03.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:03.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:03.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:03.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:03.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[26617]: from='client.24268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi116:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:53:03.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:53:03.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:53:03.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:03.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[26617]: pgmap v154: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:53:03.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[37567]: Detected new or changed devices on smithi116 2024-04-26T04:53:03.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:03.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:03.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:03.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:03.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:03.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[37567]: Adjusting osd_memory_target on smithi116 to 5696M 2024-04-26T04:53:03.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:03.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:03.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:03.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:03.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:03.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:03.516 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:03.516 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:03.516 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[37567]: from='client.24268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi116:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:53:03.516 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:53:03.516 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:53:03.516 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:03.516 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:03 smithi012 ceph-mon[37567]: pgmap v154: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:53:05.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:05 smithi116 ceph-mon[29897]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-26T04:53:05.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:05 smithi116 ceph-mon[29897]: from='client.? 172.21.15.116:0/2213913075' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "439d336e-0012-4b61-893f-50f562863965"}]: dispatch 2024-04-26T04:53:05.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:05 smithi116 ceph-mon[29897]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "439d336e-0012-4b61-893f-50f562863965"}]: dispatch 2024-04-26T04:53:05.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:05 smithi116 ceph-mon[29897]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "439d336e-0012-4b61-893f-50f562863965"}]': finished 2024-04-26T04:53:05.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:05 smithi116 ceph-mon[29897]: osdmap e52: 8 total, 7 up, 8 in 2024-04-26T04:53:05.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:05 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:53:05.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:05 smithi116 ceph-mon[29897]: from='client.? 172.21.15.116:0/2665291711' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:53:06.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:05 smithi012 ceph-mon[37567]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-26T04:53:06.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:05 smithi012 ceph-mon[37567]: from='client.? 172.21.15.116:0/2213913075' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "439d336e-0012-4b61-893f-50f562863965"}]: dispatch 2024-04-26T04:53:06.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:05 smithi012 ceph-mon[37567]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "439d336e-0012-4b61-893f-50f562863965"}]: dispatch 2024-04-26T04:53:06.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:05 smithi012 ceph-mon[37567]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "439d336e-0012-4b61-893f-50f562863965"}]': finished 2024-04-26T04:53:06.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:05 smithi012 ceph-mon[37567]: osdmap e52: 8 total, 7 up, 8 in 2024-04-26T04:53:06.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:05 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:53:06.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:05 smithi012 ceph-mon[37567]: from='client.? 172.21.15.116:0/2665291711' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:53:06.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:05 smithi012 ceph-mon[26617]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-26T04:53:06.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:05 smithi012 ceph-mon[26617]: from='client.? 172.21.15.116:0/2213913075' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "439d336e-0012-4b61-893f-50f562863965"}]: dispatch 2024-04-26T04:53:06.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:05 smithi012 ceph-mon[26617]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "439d336e-0012-4b61-893f-50f562863965"}]: dispatch 2024-04-26T04:53:06.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:05 smithi012 ceph-mon[26617]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "439d336e-0012-4b61-893f-50f562863965"}]': finished 2024-04-26T04:53:06.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:05 smithi012 ceph-mon[26617]: osdmap e52: 8 total, 7 up, 8 in 2024-04-26T04:53:06.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:05 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:53:06.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:05 smithi012 ceph-mon[26617]: from='client.? 172.21.15.116:0/2665291711' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:53:07.849 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:07 smithi116 ceph-mon[29897]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-04-26T04:53:08.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:07 smithi012 ceph-mon[26617]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-04-26T04:53:08.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:07 smithi012 ceph-mon[37567]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-04-26T04:53:09.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:09 smithi116 ceph-mon[29897]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-26T04:53:10.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:09 smithi012 ceph-mon[26617]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-26T04:53:10.012 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:09 smithi012 ceph-mon[37567]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-26T04:53:11.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:11 smithi116 ceph-mon[29897]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-04-26T04:53:11.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:11 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T04:53:11.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:11 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:11.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:11 smithi116 ceph-mon[29897]: Deploying daemon osd.7 on smithi116 2024-04-26T04:53:12.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:11 smithi012 ceph-mon[26617]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-04-26T04:53:12.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:11 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T04:53:12.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:11 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:12.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:11 smithi012 ceph-mon[26617]: Deploying daemon osd.7 on smithi116 2024-04-26T04:53:12.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:11 smithi012 ceph-mon[37567]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-04-26T04:53:12.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:11 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T04:53:12.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:11 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:12.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:11 smithi012 ceph-mon[37567]: Deploying daemon osd.7 on smithi116 2024-04-26T04:53:13.978 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:13 smithi116 ceph-mon[29897]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-26T04:53:14.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:13 smithi012 ceph-mon[26617]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-26T04:53:14.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:13 smithi012 ceph-mon[37567]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-26T04:53:15.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:15 smithi116 ceph-mon[29897]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:53:16.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:15 smithi012 ceph-mon[26617]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:53:16.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:15 smithi012 ceph-mon[37567]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:53:17.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:16 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:17.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:16 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:17.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:16 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:17.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:16 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:17.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:16 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:17.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:16 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:17.057 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:16 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:17.057 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:16 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:17.057 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:16 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:18.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:17 smithi012 ceph-mon[37567]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:53:18.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:17 smithi012 ceph-mon[26617]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:53:18.227 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:17 smithi116 ceph-mon[29897]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:53:18.503 INFO:teuthology.orchestra.run.smithi116.stdout:Created osd(s) 7 on host 'smithi116' 2024-04-26T04:53:19.434 DEBUG:teuthology.orchestra.run.smithi116:osd.7> sudo journalctl -f -n 0 -u ceph-0b9350c6-0388-11ef-bc93-c7b262605968@osd.7.service 2024-04-26T04:53:19.437 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-04-26T04:53:19.437 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph osd stat -f json 2024-04-26T04:53:19.728 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:53:19.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:19 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:19.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:19 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:19.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:19 smithi116 ceph-mon[29897]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:53:19.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:19 smithi116 ceph-mon[29897]: from='osd.7 [v2:172.21.15.116:6824/3779511167,v1:172.21.15.116:6825/3779511167]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T04:53:19.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:19 smithi116 ceph-mon[29897]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T04:53:19.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:19 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:19.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:19 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:19.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:19 smithi012 ceph-mon[37567]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:53:19.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:19 smithi012 ceph-mon[37567]: from='osd.7 [v2:172.21.15.116:6824/3779511167,v1:172.21.15.116:6825/3779511167]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T04:53:19.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:19 smithi012 ceph-mon[37567]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T04:53:19.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:19 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:19.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:19 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:19.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:19 smithi012 ceph-mon[26617]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:53:19.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:19 smithi012 ceph-mon[26617]: from='osd.7 [v2:172.21.15.116:6824/3779511167,v1:172.21.15.116:6825/3779511167]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T04:53:19.765 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:19 smithi012 ceph-mon[26617]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T04:53:20.574 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:20 smithi116 ceph-mon[29897]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-26T04:53:20.574 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:20 smithi116 ceph-mon[29897]: osdmap e53: 8 total, 7 up, 8 in 2024-04-26T04:53:20.574 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:20 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:53:20.574 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:20 smithi116 ceph-mon[29897]: from='osd.7 [v2:172.21.15.116:6824/3779511167,v1:172.21.15.116:6825/3779511167]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]: dispatch 2024-04-26T04:53:20.574 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:20 smithi116 ceph-mon[29897]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]: dispatch 2024-04-26T04:53:20.574 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:20 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:20.574 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:20 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:20.574 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 04:53:20 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T04:53:20.514+0000 7f8d9091f700 -1 osd.7 0 waiting for initial osdmap 2024-04-26T04:53:20.575 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 04:53:20 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T04:53:20.522+0000 7f8d8a718700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:53:20.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:20 smithi012 ceph-mon[37567]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-26T04:53:20.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:20 smithi012 ceph-mon[37567]: osdmap e53: 8 total, 7 up, 8 in 2024-04-26T04:53:20.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:20 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:53:20.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:20 smithi012 ceph-mon[37567]: from='osd.7 [v2:172.21.15.116:6824/3779511167,v1:172.21.15.116:6825/3779511167]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]: dispatch 2024-04-26T04:53:20.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:20 smithi012 ceph-mon[37567]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]: dispatch 2024-04-26T04:53:20.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:20 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:20.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:20 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:20.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:20 smithi012 ceph-mon[26617]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-26T04:53:20.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:20 smithi012 ceph-mon[26617]: osdmap e53: 8 total, 7 up, 8 in 2024-04-26T04:53:20.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:20 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:53:20.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:20 smithi012 ceph-mon[26617]: from='osd.7 [v2:172.21.15.116:6824/3779511167,v1:172.21.15.116:6825/3779511167]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]: dispatch 2024-04-26T04:53:20.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:20 smithi012 ceph-mon[26617]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]: dispatch 2024-04-26T04:53:20.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:20 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:20.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:20 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:21.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:21 smithi116 ceph-mon[29897]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]': finished 2024-04-26T04:53:21.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:21 smithi116 ceph-mon[29897]: osdmap e54: 8 total, 7 up, 8 in 2024-04-26T04:53:21.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:21 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:53:21.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:21 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:53:21.743 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:21 smithi116 ceph-mon[29897]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:53:21.743 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:21 smithi116 ceph-mon[29897]: osd.7 [v2:172.21.15.116:6824/3779511167,v1:172.21.15.116:6825/3779511167] boot 2024-04-26T04:53:21.743 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:21 smithi116 ceph-mon[29897]: osdmap e55: 8 total, 8 up, 8 in 2024-04-26T04:53:21.743 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:21 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:53:21.752 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:53:21.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:21 smithi012 ceph-mon[26617]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]': finished 2024-04-26T04:53:21.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:21 smithi012 ceph-mon[26617]: osdmap e54: 8 total, 7 up, 8 in 2024-04-26T04:53:21.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:21 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:53:21.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:21 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:53:21.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:21 smithi012 ceph-mon[26617]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:53:21.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:21 smithi012 ceph-mon[26617]: osd.7 [v2:172.21.15.116:6824/3779511167,v1:172.21.15.116:6825/3779511167] boot 2024-04-26T04:53:21.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:21 smithi012 ceph-mon[26617]: osdmap e55: 8 total, 8 up, 8 in 2024-04-26T04:53:21.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:21 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:53:21.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:21 smithi012 ceph-mon[37567]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]': finished 2024-04-26T04:53:21.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:21 smithi012 ceph-mon[37567]: osdmap e54: 8 total, 7 up, 8 in 2024-04-26T04:53:21.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:21 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:53:21.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:21 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:53:21.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:21 smithi012 ceph-mon[37567]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:53:21.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:21 smithi012 ceph-mon[37567]: osd.7 [v2:172.21.15.116:6824/3779511167,v1:172.21.15.116:6825/3779511167] boot 2024-04-26T04:53:21.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:21 smithi012 ceph-mon[37567]: osdmap e55: 8 total, 8 up, 8 in 2024-04-26T04:53:21.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:21 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:53:22.410 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":55,"num_osds":8,"num_up_osds":8,"osd_up_since":1714107201,"num_in_osds":8,"osd_in_since":1714107184,"num_remapped_pgs":1} 2024-04-26T04:53:22.411 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T04:53:22.677 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:53:22.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:22 smithi012 ceph-mon[26617]: purged_snaps scrub starts 2024-04-26T04:53:22.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:22 smithi012 ceph-mon[26617]: purged_snaps scrub ok 2024-04-26T04:53:22.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:22 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1815400161' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-26T04:53:22.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:22 smithi012 ceph-mon[26617]: osdmap e56: 8 total, 8 up, 8 in 2024-04-26T04:53:22.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:22 smithi012 ceph-mon[37567]: purged_snaps scrub starts 2024-04-26T04:53:22.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:22 smithi012 ceph-mon[37567]: purged_snaps scrub ok 2024-04-26T04:53:22.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:22 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/1815400161' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-26T04:53:22.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:22 smithi012 ceph-mon[37567]: osdmap e56: 8 total, 8 up, 8 in 2024-04-26T04:53:22.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:22 smithi116 ceph-mon[29897]: purged_snaps scrub starts 2024-04-26T04:53:22.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:22 smithi116 ceph-mon[29897]: purged_snaps scrub ok 2024-04-26T04:53:22.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:22 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/1815400161' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-26T04:53:22.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:22 smithi116 ceph-mon[29897]: osdmap e56: 8 total, 8 up, 8 in 2024-04-26T04:53:23.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:23 smithi012 ceph-mon[37567]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:23.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:23 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:23.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:23 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:23.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:23 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:23.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:23 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:23.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:23 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:23.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:23 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:23.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:23 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:23.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:23 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:23.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:23 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:23.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:23 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:23.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:23 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:23.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:23 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:23.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:23 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:23.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:23 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:23.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:23 smithi012 ceph-mon[37567]: osdmap e57: 8 total, 8 up, 8 in 2024-04-26T04:53:23.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:23 smithi012 ceph-mon[26617]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:23.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:23 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:23.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:23 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:23.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:23 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:23.765 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:23 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:23.765 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:23 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:23.765 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:23 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:23.765 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:23 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:23.765 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:23 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:23.765 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:23 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:23.765 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:23 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:23.765 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:23 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:23.765 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:23 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:23.765 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:23 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:23.765 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:23 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:23.765 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:23 smithi012 ceph-mon[26617]: osdmap e57: 8 total, 8 up, 8 in 2024-04-26T04:53:23.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:23 smithi116 ceph-mon[29897]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:23.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:23 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:23.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:23 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:23.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:23 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:23.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:23 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:23.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:23 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:23.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:23 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:23.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:23 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:23.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:23 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:23.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:23 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:23.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:23 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:23.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:23 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:23.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:23 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:23.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:23 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:23.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:23 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:23.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:23 smithi116 ceph-mon[29897]: osdmap e57: 8 total, 8 up, 8 in 2024-04-26T04:53:24.596 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:53:24.596 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":58,"fsid":"0b9350c6-0388-11ef-bc93-c7b262605968","created":"2024-04-26T04:48:17.966774+0000","modified":"2024-04-26T04:53:24.529008+0000","last_up_change":"2024-04-26T04:53:21.501626+0000","last_in_change":"2024-04-26T04:53:04.660871+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:51:27.066997+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"5f43d5c7-23d9-48a5-bc3c-a3a6ce3c004f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6802","nonce":1077619568},{"type":"v1","addr":"172.21.15.12:6803","nonce":1077619568}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6804","nonce":1077619568},{"type":"v1","addr":"172.21.15.12:6805","nonce":1077619568}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6808","nonce":1077619568},{"type":"v1","addr":"172.21.15.12:6809","nonce":1077619568}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6806","nonce":1077619568},{"type":"v1","addr":"172.21.15.12:6807","nonce":1077619568}]},"public_addr":"172.21.15.12:6803/1077619568","cluster_addr":"172.21.15.12:6805/1077619568","heartbeat_back_addr":"172.21.15.12:6809/1077619568","heartbeat_front_addr":"172.21.15.12:6807/1077619568","state":["exists","up"]},{"osd":1,"uuid":"75471714-6e29-4e02-9c03-86ce919b1ff0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6810","nonce":726116340},{"type":"v1","addr":"172.21.15.12:6811","nonce":726116340}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6812","nonce":726116340},{"type":"v1","addr":"172.21.15.12:6813","nonce":726116340}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6816","nonce":726116340},{"type":"v1","addr":"172.21.15.12:6817","nonce":726116340}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6814","nonce":726116340},{"type":"v1","addr":"172.21.15.12:6815","nonce":726116340}]},"public_addr":"172.21.15.12:6811/726116340","cluster_addr":"172.21.15.12:6813/726116340","heartbeat_back_addr":"172.21.15.12:6817/726116340","heartbeat_front_addr":"172.21.15.12:6815/726116340","state":["exists","up"]},{"osd":2,"uuid":"4b402201-4f28-4bc1-9cf0-0384090fb158","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6818","nonce":4118496923},{"type":"v1","addr":"172.21.15.12:6819","nonce":4118496923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6820","nonce":4118496923},{"type":"v1","addr":"172.21.15.12:6821","nonce":4118496923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6824","nonce":4118496923},{"type":"v1","addr":"172.21.15.12:6825","nonce":4118496923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6822","nonce":4118496923},{"type":"v1","addr":"172.21.15.12:6823","nonce":4118496923}]},"public_addr":"172.21.15.12:6819/4118496923","cluster_addr":"172.21.15.12:6821/4118496923","heartbeat_back_addr":"172.21.15.12:6825/4118496923","heartbeat_front_addr":"172.21.15.12:6823/4118496923","state":["exists","up"]},{"osd":3,"uuid":"54bf03b9-b2e6-4230-8dcd-d62af8853c36","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6826","nonce":3374180434},{"type":"v1","addr":"172.21.15.12:6827","nonce":3374180434}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6828","nonce":3374180434},{"type":"v1","addr":"172.21.15.12:6829","nonce":3374180434}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6832","nonce":3374180434},{"type":"v1","addr":"172.21.15.12:6833","nonce":3374180434}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6830","nonce":3374180434},{"type":"v1","addr":"172.21.15.12:6831","nonce":3374180434}]},"public_addr":"172.21.15.12:6827/3374180434","cluster_addr":"172.21.15.12:6829/3374180434","heartbeat_back_addr":"172.21.15.12:6833/3374180434","heartbeat_front_addr":"172.21.15.12:6831/3374180434","state":["exists","up"]},{"osd":4,"uuid":"ea9d3fb5-1877-4971-a8fb-8b5aafa7871a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6800","nonce":2011782592},{"type":"v1","addr":"172.21.15.116:6801","nonce":2011782592}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6802","nonce":2011782592},{"type":"v1","addr":"172.21.15.116:6803","nonce":2011782592}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6806","nonce":2011782592},{"type":"v1","addr":"172.21.15.116:6807","nonce":2011782592}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6804","nonce":2011782592},{"type":"v1","addr":"172.21.15.116:6805","nonce":2011782592}]},"public_addr":"172.21.15.116:6801/2011782592","cluster_addr":"172.21.15.116:6803/2011782592","heartbeat_back_addr":"172.21.15.116:6807/2011782592","heartbeat_front_addr":"172.21.15.116:6805/2011782592","state":["exists","up"]},{"osd":5,"uuid":"59f0530d-37eb-41d8-86d0-580ba9b8e4e2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6808","nonce":4077435109},{"type":"v1","addr":"172.21.15.116:6809","nonce":4077435109}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6810","nonce":4077435109},{"type":"v1","addr":"172.21.15.116:6811","nonce":4077435109}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6814","nonce":4077435109},{"type":"v1","addr":"172.21.15.116:6815","nonce":4077435109}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6812","nonce":4077435109},{"type":"v1","addr":"172.21.15.116:6813","nonce":4077435109}]},"public_addr":"172.21.15.116:6809/4077435109","cluster_addr":"172.21.15.116:6811/4077435109","heartbeat_back_addr":"172.21.15.116:6815/4077435109","heartbeat_front_addr":"172.21.15.116:6813/4077435109","state":["exists","up"]},{"osd":6,"uuid":"62b1edf5-75b9-4ea6-af34-d40584a6c343","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6816","nonce":2560073849},{"type":"v1","addr":"172.21.15.116:6817","nonce":2560073849}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6818","nonce":2560073849},{"type":"v1","addr":"172.21.15.116:6819","nonce":2560073849}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6822","nonce":2560073849},{"type":"v1","addr":"172.21.15.116:6823","nonce":2560073849}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6820","nonce":2560073849},{"type":"v1","addr":"172.21.15.116:6821","nonce":2560073849}]},"public_addr":"172.21.15.116:6817/2560073849","cluster_addr":"172.21.15.116:6819/2560073849","heartbeat_back_addr":"172.21.15.116:6823/2560073849","heartbeat_front_addr":"172.21.15.116:6821/2560073849","state":["exists","up"]},{"osd":7,"uuid":"439d336e-0012-4b61-893f-50f562863965","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6824","nonce":3779511167},{"type":"v1","addr":"172.21.15.116:6825","nonce":3779511167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6826","nonce":3779511167},{"type":"v1","addr":"172.21.15.116:6827","nonce":3779511167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6830","nonce":3779511167},{"type":"v1","addr":"172.21.15.116:6831","nonce":3779511167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6828","nonce":3779511167},{"type":"v1","addr":"172.21.15.116:6829","nonce":3779511167}]},"public_addr":"172.21.15.116:6825/3779511167","cluster_addr":"172.21.15.116:6827/3779511167","heartbeat_back_addr":"172.21.15.116:6831/3779511167","heartbeat_front_addr":"172.21.15.116:6829/3779511167","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:50:34.985771+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:50:58.677899+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:51:23.046413+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:51:48.496821+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:09.812658+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:32.853940+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:56.215123+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:53:19.840326+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.12:0/4212767587":"2024-04-27T04:49:16.560212+0000","172.21.15.12:0/1150032450":"2024-04-27T04:49:16.560212+0000","172.21.15.12:6801/2681852455":"2024-04-27T04:49:16.560212+0000","172.21.15.12:6801/378740862":"2024-04-27T04:48:52.473898+0000","172.21.15.12:6800/378740862":"2024-04-27T04:48:52.473898+0000","172.21.15.12:0/320935135":"2024-04-27T04:48:52.473898+0000","172.21.15.12:0/4007370348":"2024-04-27T04:49:16.560212+0000","172.21.15.12:0/2016967349":"2024-04-27T04:48:52.473898+0000","172.21.15.12:0/2418603751":"2024-04-27T04:48:52.473898+0000","172.21.15.12:6800/2681852455":"2024-04-27T04:49:16.560212+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T04:53:24.966 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:24 smithi012 ceph-mon[26617]: Detected new or changed devices on smithi116 2024-04-26T04:53:24.967 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:24 smithi012 ceph-mon[26617]: Adjusting osd_memory_target on smithi116 to 4272M 2024-04-26T04:53:24.967 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:24 smithi012 ceph-mon[37567]: Detected new or changed devices on smithi116 2024-04-26T04:53:24.967 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:24 smithi012 ceph-mon[37567]: Adjusting osd_memory_target on smithi116 to 4272M 2024-04-26T04:53:24.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:24 smithi116 ceph-mon[29897]: Detected new or changed devices on smithi116 2024-04-26T04:53:24.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:24 smithi116 ceph-mon[29897]: Adjusting osd_memory_target on smithi116 to 4272M 2024-04-26T04:53:25.364 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-26T04:51:27.066997+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-04-26T04:53:25.364 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-04-26T04:53:25.632 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:53:25.754 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:25 smithi012 ceph-mon[37567]: osdmap e58: 8 total, 8 up, 8 in 2024-04-26T04:53:25.755 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:25 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/3699920897' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:53:25.755 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:25 smithi012 ceph-mon[37567]: pgmap v172: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:25.755 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:25 smithi012 ceph-mon[26617]: osdmap e58: 8 total, 8 up, 8 in 2024-04-26T04:53:25.755 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:25 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/3699920897' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:53:25.755 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:25 smithi012 ceph-mon[26617]: pgmap v172: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:25.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:25 smithi116 ceph-mon[29897]: osdmap e58: 8 total, 8 up, 8 in 2024-04-26T04:53:25.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:25 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/3699920897' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:53:25.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:25 smithi116 ceph-mon[29897]: pgmap v172: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:26.917 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:26 smithi012 ceph-mon[26617]: osdmap e59: 8 total, 8 up, 8 in 2024-04-26T04:53:26.917 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:26 smithi012 ceph-mon[37567]: osdmap e59: 8 total, 8 up, 8 in 2024-04-26T04:53:26.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:26 smithi116 ceph-mon[29897]: osdmap e59: 8 total, 8 up, 8 in 2024-04-26T04:53:27.479 INFO:teuthology.orchestra.run.smithi012.stdout:pg_num: 1 2024-04-26T04:53:27.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:27 smithi012 ceph-mon[26617]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 141 KiB/s, 0 objects/s recovering 2024-04-26T04:53:27.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:27 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/2578522306' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-26T04:53:27.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:27 smithi012 ceph-mon[37567]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 141 KiB/s, 0 objects/s recovering 2024-04-26T04:53:27.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:27 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/2578522306' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-26T04:53:27.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:27 smithi116 ceph-mon[29897]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 141 KiB/s, 0 objects/s recovering 2024-04-26T04:53:27.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:27 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/2578522306' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-26T04:53:28.059 INFO:tasks.cephadm:Adding prometheus.a on smithi116 2024-04-26T04:53:28.060 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph orch apply prometheus '1;smithi116=a' 2024-04-26T04:53:28.320 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:53:29.908 INFO:teuthology.orchestra.run.smithi116.stdout:Scheduled prometheus update... 2024-04-26T04:53:29.909 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:29 smithi116 ceph-mon[29897]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-26T04:53:30.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:29 smithi012 ceph-mon[26617]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-26T04:53:30.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:29 smithi012 ceph-mon[37567]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-26T04:53:30.564 DEBUG:teuthology.orchestra.run.smithi116:prometheus.a> sudo journalctl -f -n 0 -u ceph-0b9350c6-0388-11ef-bc93-c7b262605968@prometheus.a.service 2024-04-26T04:53:30.566 INFO:tasks.cephadm:Adding node-exporter.a on smithi012 2024-04-26T04:53:30.567 INFO:tasks.cephadm:Adding node-exporter.b on smithi116 2024-04-26T04:53:30.567 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph orch apply node-exporter '2;smithi012=a;smithi116=b' 2024-04-26T04:53:30.867 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:53:31.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:30 smithi116 ceph-mon[29897]: from='client.24307 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi116=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:53:31.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:30 smithi116 ceph-mon[29897]: Saving service prometheus spec with placement smithi116=a;count:1 2024-04-26T04:53:31.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:30 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:31.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:30 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:31.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:30 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:31.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:30 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:31.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:30 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:31.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:30 smithi116 ceph-mon[29897]: Deploying daemon prometheus.a on smithi116 2024-04-26T04:53:31.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:30 smithi116 ceph-mon[29897]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-04-26T04:53:31.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:30 smithi012 ceph-mon[26617]: from='client.24307 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi116=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:53:31.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:30 smithi012 ceph-mon[26617]: Saving service prometheus spec with placement smithi116=a;count:1 2024-04-26T04:53:31.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:30 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:31.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:30 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:31.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:30 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:31.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:30 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:31.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:30 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:31.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:30 smithi012 ceph-mon[26617]: Deploying daemon prometheus.a on smithi116 2024-04-26T04:53:31.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:30 smithi012 ceph-mon[26617]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-04-26T04:53:31.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:30 smithi012 ceph-mon[37567]: from='client.24307 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi116=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:53:31.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:30 smithi012 ceph-mon[37567]: Saving service prometheus spec with placement smithi116=a;count:1 2024-04-26T04:53:31.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:30 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:31.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:30 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:31.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:30 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:31.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:30 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:31.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:30 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:31.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:30 smithi012 ceph-mon[37567]: Deploying daemon prometheus.a on smithi116 2024-04-26T04:53:31.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:30 smithi012 ceph-mon[37567]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-04-26T04:53:33.224 INFO:teuthology.orchestra.run.smithi116.stdout:Scheduled node-exporter update... 2024-04-26T04:53:33.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:33 smithi116 ceph-mon[29897]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-04-26T04:53:33.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:33 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:34.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:33 smithi012 ceph-mon[37567]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-04-26T04:53:34.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:33 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:34.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:33 smithi012 ceph-mon[26617]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-04-26T04:53:34.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:33 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:34.185 DEBUG:teuthology.orchestra.run.smithi012:node-exporter.a> sudo journalctl -f -n 0 -u ceph-0b9350c6-0388-11ef-bc93-c7b262605968@node-exporter.a.service 2024-04-26T04:53:34.187 DEBUG:teuthology.orchestra.run.smithi116:node-exporter.b> sudo journalctl -f -n 0 -u ceph-0b9350c6-0388-11ef-bc93-c7b262605968@node-exporter.b.service 2024-04-26T04:53:34.190 INFO:tasks.cephadm:Adding alertmanager.a on smithi012 2024-04-26T04:53:34.190 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph orch apply alertmanager '1;smithi012=a' 2024-04-26T04:53:34.479 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:53:34.936 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:34 smithi116 ceph-mon[29897]: from='client.24313 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi012=a;smithi116=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:53:34.936 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:34 smithi116 ceph-mon[29897]: Saving service node-exporter spec with placement smithi012=a;smithi116=b;count:2 2024-04-26T04:53:35.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:34 smithi012 ceph-mon[37567]: from='client.24313 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi012=a;smithi116=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:53:35.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:34 smithi012 ceph-mon[37567]: Saving service node-exporter spec with placement smithi012=a;smithi116=b;count:2 2024-04-26T04:53:35.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:34 smithi012 ceph-mon[26617]: from='client.24313 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi012=a;smithi116=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:53:35.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:34 smithi012 ceph-mon[26617]: Saving service node-exporter spec with placement smithi012=a;smithi116=b;count:2 2024-04-26T04:53:35.942 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:35 smithi116 ceph-mon[29897]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-26T04:53:36.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:35 smithi012 ceph-mon[26617]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-26T04:53:36.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:35 smithi012 ceph-mon[37567]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-26T04:53:37.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:37 smithi116 ceph-mon[29897]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-26T04:53:38.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:37 smithi012 ceph-mon[26617]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-26T04:53:38.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:37 smithi012 ceph-mon[37567]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-26T04:53:38.862 INFO:teuthology.orchestra.run.smithi116.stdout:Scheduled alertmanager update... 2024-04-26T04:53:40.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:39 smithi116 ceph-mon[29897]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:40.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:39 smithi116 ceph-mon[29897]: from='client.24319 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi012=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:53:40.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:39 smithi116 ceph-mon[29897]: Saving service alertmanager spec with placement smithi012=a;count:1 2024-04-26T04:53:40.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:39 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:40.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:39 smithi012 ceph-mon[37567]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:40.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:39 smithi012 ceph-mon[37567]: from='client.24319 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi012=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:53:40.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:39 smithi012 ceph-mon[37567]: Saving service alertmanager spec with placement smithi012=a;count:1 2024-04-26T04:53:40.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:39 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:40.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:39 smithi012 ceph-mon[26617]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:40.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:39 smithi012 ceph-mon[26617]: from='client.24319 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi012=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:53:40.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:39 smithi012 ceph-mon[26617]: Saving service alertmanager spec with placement smithi012=a;count:1 2024-04-26T04:53:40.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:39 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:40.385 DEBUG:teuthology.orchestra.run.smithi012:alertmanager.a> sudo journalctl -f -n 0 -u ceph-0b9350c6-0388-11ef-bc93-c7b262605968@alertmanager.a.service 2024-04-26T04:53:40.388 INFO:tasks.cephadm:Adding grafana.a on smithi116 2024-04-26T04:53:40.388 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph orch apply grafana '1;smithi116=a' 2024-04-26T04:53:42.186 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:41 smithi116 ceph-mon[29897]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:42.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:41 smithi012 ceph-mon[26617]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:42.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:41 smithi012 ceph-mon[37567]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:42.307 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:53:44.107 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:43 smithi116 ceph-mon[29897]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:44.262 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:43 smithi012 ceph-mon[37567]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:44.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:43 smithi012 ceph-mon[26617]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:44.483 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:53:44 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[57037]: ts=2024-04-26T04:53:44.461Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-26T04:53:44.565 INFO:teuthology.orchestra.run.smithi116.stdout:Scheduled grafana update... 2024-04-26T04:53:44.741 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:53:44 smithi116 bash[57018]: 36962f3d5221808c45119b01b4c65a2d9816fc2ff4379115fe571bbf592bdd89 2024-04-26T04:53:44.741 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:53:44 smithi116 systemd[1]: Started Ceph prometheus.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:53:45.229 DEBUG:teuthology.orchestra.run.smithi116:grafana.a> sudo journalctl -f -n 0 -u ceph-0b9350c6-0388-11ef-bc93-c7b262605968@grafana.a.service 2024-04-26T04:53:45.231 INFO:tasks.cephadm:Setting up client nodes... 2024-04-26T04:53:45.232 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-26T04:53:45.505 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:53:45.892 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:45 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:45.892 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:45 smithi012 ceph-mon[26617]: from='client.24325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi116=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:53:45.893 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:45 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:45.893 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:45 smithi012 ceph-mon[26617]: Saving service grafana spec with placement smithi116=a;count:1 2024-04-26T04:53:45.893 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:45 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:45.893 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:45 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:45.893 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:45 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-26T04:53:45.893 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:45 smithi012 ceph-mon[26617]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:45.893 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:45 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:45.893 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:45 smithi012 ceph-mon[37567]: from='client.24325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi116=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:53:45.893 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:45 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:45.893 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:45 smithi012 ceph-mon[37567]: Saving service grafana spec with placement smithi116=a;count:1 2024-04-26T04:53:45.894 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:45 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:45.894 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:45 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:45.894 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:45 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-26T04:53:45.894 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:45 smithi012 ceph-mon[37567]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:45.894 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:53:45 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: ignoring --setuser ceph since I am not root 2024-04-26T04:53:45.894 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:53:45 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: ignoring --setgroup ceph since I am not root 2024-04-26T04:53:45.894 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:53:45 smithi012 ceph-mgr[26852]: -- 172.21.15.12:0/2927009643 <== mon.0 v2:172.21.15.12:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56050b05c9c0 con 0x56050b033000 2024-04-26T04:53:45.894 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:53:45 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:53:45.759+0000 7fdbadee71c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:53:45.908 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:53:45 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: ignoring --setuser ceph since I am not root 2024-04-26T04:53:45.909 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:53:45 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: ignoring --setgroup ceph since I am not root 2024-04-26T04:53:45.909 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:53:45 smithi116 ceph-mgr[31814]: -- 172.21.15.116:0/2276603322 <== mon.1 v2:172.21.15.116:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x558e46d929c0 con 0x558e46d68c00 2024-04-26T04:53:45.909 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:53:45 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:53:45.751+0000 7f20c1dc41c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:53:45.909 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:53:45 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:53:45.907+0000 7f20c1dc41c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:53:45.909 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:45 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:45.909 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:45 smithi116 ceph-mon[29897]: from='client.24325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi116=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:53:45.909 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:45 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:45.909 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:45 smithi116 ceph-mon[29897]: Saving service grafana spec with placement smithi116=a;count:1 2024-04-26T04:53:45.909 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:45 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:45.909 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:45 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' 2024-04-26T04:53:45.909 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:45 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-26T04:53:45.909 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:45 smithi116 ceph-mon[29897]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:46.241 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:53:46 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:53:46.008+0000 7f20c1dc41c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:53:46.262 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:53:45 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:53:45.934+0000 7fdbadee71c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:53:46.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:53:46 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:53:46.038+0000 7fdbadee71c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:53:46.591 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:53:46 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:53:46.309+0000 7f20c1dc41c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:53:46.592 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:53:46 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:53:46.390+0000 7f20c1dc41c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:53:46.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:46 smithi012 ceph-mon[26617]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-26T04:53:46.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:46 smithi012 ceph-mon[26617]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-26T04:53:46.685 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:46 smithi012 ceph-mon[37567]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-26T04:53:46.686 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:46 smithi012 ceph-mon[37567]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-26T04:53:46.686 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:53:46 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:53:46.358+0000 7fdbadee71c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:53:46.686 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:53:46 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:53:46.443+0000 7fdbadee71c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:53:46.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:46 smithi116 ceph-mon[29897]: from='mgr.14152 172.21.15.12:0/387977090' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-26T04:53:46.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:46 smithi116 ceph-mon[29897]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-26T04:53:47.526 INFO:teuthology.orchestra.run.smithi012.stdout:[client.0] 2024-04-26T04:53:47.526 INFO:teuthology.orchestra.run.smithi012.stdout: key = AQBbMytmCFgmHxAAlllOeAJ5t3gC34GPMS4j/A== 2024-04-26T04:53:47.843 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:47 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/1462023691' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:53:47.843 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:47 smithi116 ceph-mon[29897]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:53:47.843 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:47 smithi116 ceph-mon[29897]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T04:53:47.843 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:53:47 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:53:47.515+0000 7f20c1dc41c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:53:47.843 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:53:47 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:53:47.677+0000 7f20c1dc41c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:53:47.854 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:47 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1462023691' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:53:47.854 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:47 smithi012 ceph-mon[26617]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:53:47.854 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:47 smithi012 ceph-mon[26617]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T04:53:47.854 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:53:47 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:53:47.680+0000 7fdbadee71c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:53:47.854 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:47 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/1462023691' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:53:47.854 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:47 smithi012 ceph-mon[37567]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:53:47.855 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:47 smithi012 ceph-mon[37567]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T04:53:48.100 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:53:47 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:53:47.842+0000 7f20c1dc41c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:53:48.213 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:53:47 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:53:47.853+0000 7fdbadee71c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:53:48.213 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:53:48 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:53:48.027+0000 7fdbadee71c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:53:48.239 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-04-26T04:53:48.239 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-04-26T04:53:48.239 DEBUG:teuthology.orchestra.run.smithi012:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-04-26T04:53:48.276 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-26T04:53:48.491 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:53:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:53:48.099+0000 7f20c1dc41c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:53:48.491 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:53:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:53:48.188+0000 7f20c1dc41c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:53:48.491 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:53:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:53:48.279+0000 7f20c1dc41c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:53:48.498 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:53:48 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:53:48.307+0000 7fdbadee71c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:53:48.498 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:53:48 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:53:48.400+0000 7fdbadee71c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:53:48.498 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:53:48 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:53:48.497+0000 7fdbadee71c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:53:48.545 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:53:48.866 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:53:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:53:48.589+0000 7f20c1dc41c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:53:49.241 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:53:49 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:53:49.056+0000 7f20c1dc41c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:53:49.262 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:53:48 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:53:48.785+0000 7fdbadee71c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:53:49.762 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:53:49 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:53:49.265+0000 7fdbadee71c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:53:49.857 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:53:49 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:53:49.582+0000 7f20c1dc41c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:53:49.857 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:53:49 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:53:49.677+0000 7f20c1dc41c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:53:49.857 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:53:49 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:53:49.856+0000 7f20c1dc41c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:53:50.097 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:53:49 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:53:49.813+0000 7fdbadee71c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:53:50.097 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:53:49 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:53:49.910+0000 7fdbadee71c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:53:50.097 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:53:50 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:53:50.096+0000 7fdbadee71c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:53:50.404 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:53:50 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:53:50.147+0000 7f20c1dc41c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:53:50.669 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:53:50 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:53:50.402+0000 7fdbadee71c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:53:50.741 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:53:50 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:53:50.403+0000 7f20c1dc41c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:53:50.741 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:53:50 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:53:50.641+0000 7f20c1dc41c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:53:51.012 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:53:50 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:53:50.668+0000 7fdbadee71c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:53:51.013 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:53:50 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:53:50.916+0000 7fdbadee71c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:53:51.068 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:53:50 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:53:50.772+0000 7f20c1dc41c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:53:51.068 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:53:50 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:53:50.865+0000 7f20c1dc41c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:53:51.068 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:53:50 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:53:50.979+0000 7f20c1dc41c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:53:51.069 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:53:51 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:53:51.067+0000 7f20c1dc41c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:53:51.363 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:53:51 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:53:51.053+0000 7fdbadee71c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:53:51.363 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:53:51 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:53:51.150+0000 7fdbadee71c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:53:51.363 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:53:51 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:53:51.269+0000 7fdbadee71c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:53:51.363 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:53:51 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:53:51.362+0000 7fdbadee71c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:53:51.947 INFO:teuthology.orchestra.run.smithi116.stdout:[client.1] 2024-04-26T04:53:51.948 INFO:teuthology.orchestra.run.smithi116.stdout: key = AQBfMytmvVw1OBAAdrUWs7V3v+0z4hJHqrPPsA== 2024-04-26T04:53:52.193 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:51 smithi116 ceph-mon[29897]: from='client.? 172.21.15.116:0/162068408' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:53:52.193 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:51 smithi116 ceph-mon[29897]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:53:52.193 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:51 smithi116 ceph-mon[29897]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T04:53:52.193 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:53:51 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:53:51.884+0000 7f20c1dc41c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:53:52.193 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:53:52.084+0000 7f20c1dc41c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:53:52.193 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: [26/Apr/2024:04:53:52] ENGINE Bus STARTING 2024-04-26T04:53:52.193 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: CherryPy Checker: 2024-04-26T04:53:52.193 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: The Application mounted at '' has an empty config. 2024-04-26T04:53:52.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:51 smithi012 ceph-mon[26617]: from='client.? 172.21.15.116:0/162068408' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:53:52.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:51 smithi012 ceph-mon[26617]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:53:52.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:51 smithi012 ceph-mon[26617]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T04:53:52.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:51 smithi012 ceph-mon[37567]: from='client.? 172.21.15.116:0/162068408' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:53:52.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:51 smithi012 ceph-mon[37567]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:53:52.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:51 smithi012 ceph-mon[37567]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T04:53:52.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:53:52.192+0000 7fdbadee71c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:53:52.491 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: 2024-04-26T04:53:52.491 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: [26/Apr/2024:04:53:52] ENGINE Serving on http://:::9283 2024-04-26T04:53:52.491 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: [26/Apr/2024:04:53:52] ENGINE Bus STARTED 2024-04-26T04:53:52.619 DEBUG:teuthology.orchestra.run.smithi116:> set -ex 2024-04-26T04:53:52.619 DEBUG:teuthology.orchestra.run.smithi116:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-04-26T04:53:52.620 DEBUG:teuthology.orchestra.run.smithi116:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-04-26T04:53:52.653 INFO:tasks.cephadm:Waiting for OSDs to come up 2024-04-26T04:53:52.653 DEBUG:teuthology.orchestra.run.smithi012:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-04-26T04:53:52.763 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:53:52.392+0000 7fdbadee71c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:53:52.763 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: [26/Apr/2024:04:53:52] ENGINE Bus STARTING 2024-04-26T04:53:52.763 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: CherryPy Checker: 2024-04-26T04:53:52.763 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: The Application mounted at '' has an empty config. 2024-04-26T04:53:52.763 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:53:52.763 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: [26/Apr/2024:04:53:52] ENGINE Serving on http://:::9283 2024-04-26T04:53:52.763 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: [26/Apr/2024:04:53:52] ENGINE Bus STARTED 2024-04-26T04:53:52.959 INFO:teuthology.misc.health.smithi012.stdout: 2024-04-26T04:53:52.959 INFO:teuthology.misc.health.smithi012.stdout:{"epoch":60,"fsid":"0b9350c6-0388-11ef-bc93-c7b262605968","created":"2024-04-26T04:48:17.966774+0000","modified":"2024-04-26T04:53:52.394910+0000","last_up_change":"2024-04-26T04:53:21.501626+0000","last_in_change":"2024-04-26T04:53:04.660871+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:51:27.066997+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"5f43d5c7-23d9-48a5-bc3c-a3a6ce3c004f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6802","nonce":1077619568},{"type":"v1","addr":"172.21.15.12:6803","nonce":1077619568}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6804","nonce":1077619568},{"type":"v1","addr":"172.21.15.12:6805","nonce":1077619568}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6808","nonce":1077619568},{"type":"v1","addr":"172.21.15.12:6809","nonce":1077619568}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6806","nonce":1077619568},{"type":"v1","addr":"172.21.15.12:6807","nonce":1077619568}]},"public_addr":"172.21.15.12:6803/1077619568","cluster_addr":"172.21.15.12:6805/1077619568","heartbeat_back_addr":"172.21.15.12:6809/1077619568","heartbeat_front_addr":"172.21.15.12:6807/1077619568","state":["exists","up"]},{"osd":1,"uuid":"75471714-6e29-4e02-9c03-86ce919b1ff0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6810","nonce":726116340},{"type":"v1","addr":"172.21.15.12:6811","nonce":726116340}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6812","nonce":726116340},{"type":"v1","addr":"172.21.15.12:6813","nonce":726116340}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6816","nonce":726116340},{"type":"v1","addr":"172.21.15.12:6817","nonce":726116340}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6814","nonce":726116340},{"type":"v1","addr":"172.21.15.12:6815","nonce":726116340}]},"public_addr":"172.21.15.12:6811/726116340","cluster_addr":"172.21.15.12:6813/726116340","heartbeat_back_addr":"172.21.15.12:6817/726116340","heartbeat_front_addr":"172.21.15.12:6815/726116340","state":["exists","up"]},{"osd":2,"uuid":"4b402201-4f28-4bc1-9cf0-0384090fb158","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6818","nonce":4118496923},{"type":"v1","addr":"172.21.15.12:6819","nonce":4118496923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6820","nonce":4118496923},{"type":"v1","addr":"172.21.15.12:6821","nonce":4118496923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6824","nonce":4118496923},{"type":"v1","addr":"172.21.15.12:6825","nonce":4118496923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6822","nonce":4118496923},{"type":"v1","addr":"172.21.15.12:6823","nonce":4118496923}]},"public_addr":"172.21.15.12:6819/4118496923","cluster_addr":"172.21.15.12:6821/4118496923","heartbeat_back_addr":"172.21.15.12:6825/4118496923","heartbeat_front_addr":"172.21.15.12:6823/4118496923","state":["exists","up"]},{"osd":3,"uuid":"54bf03b9-b2e6-4230-8dcd-d62af8853c36","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6826","nonce":3374180434},{"type":"v1","addr":"172.21.15.12:6827","nonce":3374180434}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6828","nonce":3374180434},{"type":"v1","addr":"172.21.15.12:6829","nonce":3374180434}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6832","nonce":3374180434},{"type":"v1","addr":"172.21.15.12:6833","nonce":3374180434}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6830","nonce":3374180434},{"type":"v1","addr":"172.21.15.12:6831","nonce":3374180434}]},"public_addr":"172.21.15.12:6827/3374180434","cluster_addr":"172.21.15.12:6829/3374180434","heartbeat_back_addr":"172.21.15.12:6833/3374180434","heartbeat_front_addr":"172.21.15.12:6831/3374180434","state":["exists","up"]},{"osd":4,"uuid":"ea9d3fb5-1877-4971-a8fb-8b5aafa7871a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6800","nonce":2011782592},{"type":"v1","addr":"172.21.15.116:6801","nonce":2011782592}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6802","nonce":2011782592},{"type":"v1","addr":"172.21.15.116:6803","nonce":2011782592}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6806","nonce":2011782592},{"type":"v1","addr":"172.21.15.116:6807","nonce":2011782592}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6804","nonce":2011782592},{"type":"v1","addr":"172.21.15.116:6805","nonce":2011782592}]},"public_addr":"172.21.15.116:6801/2011782592","cluster_addr":"172.21.15.116:6803/2011782592","heartbeat_back_addr":"172.21.15.116:6807/2011782592","heartbeat_front_addr":"172.21.15.116:6805/2011782592","state":["exists","up"]},{"osd":5,"uuid":"59f0530d-37eb-41d8-86d0-580ba9b8e4e2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6808","nonce":4077435109},{"type":"v1","addr":"172.21.15.116:6809","nonce":4077435109}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6810","nonce":4077435109},{"type":"v1","addr":"172.21.15.116:6811","nonce":4077435109}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6814","nonce":4077435109},{"type":"v1","addr":"172.21.15.116:6815","nonce":4077435109}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6812","nonce":4077435109},{"type":"v1","addr":"172.21.15.116:6813","nonce":4077435109}]},"public_addr":"172.21.15.116:6809/4077435109","cluster_addr":"172.21.15.116:6811/4077435109","heartbeat_back_addr":"172.21.15.116:6815/4077435109","heartbeat_front_addr":"172.21.15.116:6813/4077435109","state":["exists","up"]},{"osd":6,"uuid":"62b1edf5-75b9-4ea6-af34-d40584a6c343","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6816","nonce":2560073849},{"type":"v1","addr":"172.21.15.116:6817","nonce":2560073849}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6818","nonce":2560073849},{"type":"v1","addr":"172.21.15.116:6819","nonce":2560073849}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6822","nonce":2560073849},{"type":"v1","addr":"172.21.15.116:6823","nonce":2560073849}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6820","nonce":2560073849},{"type":"v1","addr":"172.21.15.116:6821","nonce":2560073849}]},"public_addr":"172.21.15.116:6817/2560073849","cluster_addr":"172.21.15.116:6819/2560073849","heartbeat_back_addr":"172.21.15.116:6823/2560073849","heartbeat_front_addr":"172.21.15.116:6821/2560073849","state":["exists","up"]},{"osd":7,"uuid":"439d336e-0012-4b61-893f-50f562863965","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6824","nonce":3779511167},{"type":"v1","addr":"172.21.15.116:6825","nonce":3779511167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6826","nonce":3779511167},{"type":"v1","addr":"172.21.15.116:6827","nonce":3779511167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6830","nonce":3779511167},{"type":"v1","addr":"172.21.15.116:6831","nonce":3779511167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6828","nonce":3779511167},{"type":"v1","addr":"172.21.15.116:6829","nonce":3779511167}]},"public_addr":"172.21.15.116:6825/3779511167","cluster_addr":"172.21.15.116:6827/3779511167","heartbeat_back_addr":"172.21.15.116:6831/3779511167","heartbeat_front_addr":"172.21.15.116:6829/3779511167","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:50:34.985771+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:50:58.677899+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:51:23.046413+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:51:48.496821+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:09.812658+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:32.853940+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:56.215123+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:53:19.840326+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.12:0/625629550":"2024-04-27T04:53:52.394847+0000","172.21.15.12:0/824190310":"2024-04-27T04:53:52.394847+0000","172.21.15.12:0/2421619470":"2024-04-27T04:53:52.394847+0000","172.21.15.12:6801/1235528000":"2024-04-27T04:53:52.394847+0000","172.21.15.12:0/4212767587":"2024-04-27T04:49:16.560212+0000","172.21.15.12:0/1150032450":"2024-04-27T04:49:16.560212+0000","172.21.15.12:6801/2681852455":"2024-04-27T04:49:16.560212+0000","172.21.15.12:6800/1235528000":"2024-04-27T04:53:52.394847+0000","172.21.15.12:6801/378740862":"2024-04-27T04:48:52.473898+0000","172.21.15.12:6800/378740862":"2024-04-27T04:48:52.473898+0000","172.21.15.12:0/320935135":"2024-04-27T04:48:52.473898+0000","172.21.15.12:0/4007370348":"2024-04-27T04:49:16.560212+0000","172.21.15.12:0/2016967349":"2024-04-27T04:48:52.473898+0000","172.21.15.12:0/2418603751":"2024-04-27T04:48:52.473898+0000","172.21.15.12:0/1087977851":"2024-04-27T04:53:52.394847+0000","172.21.15.12:6800/2681852455":"2024-04-27T04:49:16.560212+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T04:53:52.972 DEBUG:teuthology.misc:8 of 8 OSDs are up 2024-04-26T04:53:52.972 INFO:tasks.cephadm:Creating RBD pool 2024-04-26T04:53:52.973 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- sudo ceph --cluster ceph osd pool create rbd 8 2024-04-26T04:53:53.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[26617]: Standby manager daemon x restarted 2024-04-26T04:53:53.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[26617]: Standby manager daemon x started 2024-04-26T04:53:53.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[26617]: from='mgr.? 172.21.15.116:0/2927615101' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:53:53.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[26617]: from='mgr.? 172.21.15.116:0/2927615101' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:53:53.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[26617]: from='mgr.? 172.21.15.116:0/2927615101' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:53:53.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[26617]: from='mgr.? 172.21.15.116:0/2927615101' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:53:53.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[26617]: Active manager daemon y restarted 2024-04-26T04:53:53.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[26617]: Activating manager daemon y 2024-04-26T04:53:53.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[26617]: osdmap e60: 8 total, 8 up, 8 in 2024-04-26T04:53:53.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[26617]: mgrmap e17: y(active, starting, since 0.00666085s), standbys: x 2024-04-26T04:53:53.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:53:53.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:53:53.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:53:53.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:53:53.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:53:53.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:53:53.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:53:53.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:53:53.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:53:53.265 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:53:53.265 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:53:53.265 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:53:53.265 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:53:53.265 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:53:53.265 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:53:53.265 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:53:53.265 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[26617]: Manager daemon y is now available 2024-04-26T04:53:53.265 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:53.265 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:53.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[37567]: Standby manager daemon x restarted 2024-04-26T04:53:53.266 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[37567]: Standby manager daemon x started 2024-04-26T04:53:53.266 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[37567]: from='mgr.? 172.21.15.116:0/2927615101' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:53:53.266 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[37567]: from='mgr.? 172.21.15.116:0/2927615101' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:53:53.266 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[37567]: from='mgr.? 172.21.15.116:0/2927615101' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:53:53.266 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[37567]: from='mgr.? 172.21.15.116:0/2927615101' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:53:53.266 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[37567]: Active manager daemon y restarted 2024-04-26T04:53:53.266 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[37567]: Activating manager daemon y 2024-04-26T04:53:53.266 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[37567]: osdmap e60: 8 total, 8 up, 8 in 2024-04-26T04:53:53.266 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[37567]: mgrmap e17: y(active, starting, since 0.00666085s), standbys: x 2024-04-26T04:53:53.266 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:53:53.266 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:53:53.266 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:53:53.266 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:53:53.267 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:53:53.267 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:53:53.267 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:53:53.267 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:53:53.267 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:53:53.267 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:53:53.267 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:53:53.267 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:53:53.267 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:53:53.267 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:53:53.267 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:53:53.267 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:53:53.267 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[37567]: Manager daemon y is now available 2024-04-26T04:53:53.267 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:53.267 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:53.268 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:53.268 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:53.268 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:53:53.268 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:53:53.268 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:53:53.268 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:53:53.268 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/216367472' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:53:53.268 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:53.268 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:53.269 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:53:53.269 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:53:53.269 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:53:53.269 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:53:53.269 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:52 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/216367472' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:53:53.276 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-mon[29897]: Standby manager daemon x restarted 2024-04-26T04:53:53.276 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-mon[29897]: Standby manager daemon x started 2024-04-26T04:53:53.276 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-mon[29897]: from='mgr.? 172.21.15.116:0/2927615101' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:53:53.276 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-mon[29897]: from='mgr.? 172.21.15.116:0/2927615101' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:53:53.276 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-mon[29897]: from='mgr.? 172.21.15.116:0/2927615101' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:53:53.276 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-mon[29897]: from='mgr.? 172.21.15.116:0/2927615101' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:53:53.276 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-mon[29897]: Active manager daemon y restarted 2024-04-26T04:53:53.276 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-mon[29897]: Activating manager daemon y 2024-04-26T04:53:53.276 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-mon[29897]: osdmap e60: 8 total, 8 up, 8 in 2024-04-26T04:53:53.277 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-mon[29897]: mgrmap e17: y(active, starting, since 0.00666085s), standbys: x 2024-04-26T04:53:53.277 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:53:53.277 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:53:53.277 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:53:53.277 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:53:53.277 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:53:53.277 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:53:53.277 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:53:53.277 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:53:53.277 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:53:53.277 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:53:53.277 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:53:53.277 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:53:53.277 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:53:53.278 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:53:53.278 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:53:53.278 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:53:53.278 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-mon[29897]: Manager daemon y is now available 2024-04-26T04:53:53.278 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:53.278 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:53.278 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:53.278 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:53.278 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:53:53.278 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:53:53.278 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:53:53.278 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:53:53.278 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:52 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/216367472' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:53:53.741 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:53:53 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:53:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:53:54.097 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:53:54.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:54 smithi012 ceph-mon[26617]: [26/Apr/2024:04:53:53] ENGINE Bus STARTING 2024-04-26T04:53:54.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:54 smithi012 ceph-mon[26617]: [26/Apr/2024:04:53:53] ENGINE Serving on https://172.21.15.12:7150 2024-04-26T04:53:54.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:54 smithi012 ceph-mon[26617]: [26/Apr/2024:04:53:53] ENGINE Bus STARTED 2024-04-26T04:53:54.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:54 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:54.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:54 smithi012 ceph-mon[26617]: mgrmap e18: y(active, since 1.01029s), standbys: x 2024-04-26T04:53:54.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:54 smithi012 ceph-mon[26617]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:54.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:54 smithi012 ceph-mon[37567]: [26/Apr/2024:04:53:53] ENGINE Bus STARTING 2024-04-26T04:53:54.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:54 smithi012 ceph-mon[37567]: [26/Apr/2024:04:53:53] ENGINE Serving on https://172.21.15.12:7150 2024-04-26T04:53:54.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:54 smithi012 ceph-mon[37567]: [26/Apr/2024:04:53:53] ENGINE Bus STARTED 2024-04-26T04:53:54.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:54 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:54.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:54 smithi012 ceph-mon[37567]: mgrmap e18: y(active, since 1.01029s), standbys: x 2024-04-26T04:53:54.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:54 smithi012 ceph-mon[37567]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:54.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:54 smithi116 ceph-mon[29897]: [26/Apr/2024:04:53:53] ENGINE Bus STARTING 2024-04-26T04:53:54.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:54 smithi116 ceph-mon[29897]: [26/Apr/2024:04:53:53] ENGINE Serving on https://172.21.15.12:7150 2024-04-26T04:53:54.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:54 smithi116 ceph-mon[29897]: [26/Apr/2024:04:53:53] ENGINE Bus STARTED 2024-04-26T04:53:54.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:54 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:54.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:54 smithi116 ceph-mon[29897]: mgrmap e18: y(active, since 1.01029s), standbys: x 2024-04-26T04:53:54.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:54 smithi116 ceph-mon[29897]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:55.512 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:55 smithi012 ceph-mon[26617]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:55.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:55 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:55.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:55 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:55.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:55 smithi012 ceph-mon[26617]: mgrmap e19: y(active, since 3s), standbys: x 2024-04-26T04:53:55.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:55 smithi012 ceph-mon[37567]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:55.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:55 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:55.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:55 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:55.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:55 smithi012 ceph-mon[37567]: mgrmap e19: y(active, since 3s), standbys: x 2024-04-26T04:53:55.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:55 smithi116 ceph-mon[29897]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:55.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:55 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:55.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:55 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:55.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:55 smithi116 ceph-mon[29897]: mgrmap e19: y(active, since 3s), standbys: x 2024-04-26T04:53:56.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:56 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:56.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:56 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:56.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:56 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:56.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:56 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:56.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:56 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:56.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:56 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:56.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:56 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:56.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:56 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:56.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:56 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:56.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:56 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:56.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:56 smithi116 ceph-mon[29897]: Adjusting osd_memory_target on smithi116 to 4016M 2024-04-26T04:53:56.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:56 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:56.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:56 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:56.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:56 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:56.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:56 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:56.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:56 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:56.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:56 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:56.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:56 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:56.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:56 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:56.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:56 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:56.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:56 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:56.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:56 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:56.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:56 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:56.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:56 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:56.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:56 smithi012 ceph-mon[26617]: Adjusting osd_memory_target on smithi116 to 4016M 2024-04-26T04:53:56.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:56 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:56.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:56 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:56.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:56 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:56.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:56 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:56.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:56 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:56.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:56 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:56.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:56 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:56.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:56 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:56.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:56 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:56.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:56 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:56.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:56 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:56.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:56 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:56.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:56 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:53:56.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:56 smithi012 ceph-mon[37567]: Adjusting osd_memory_target on smithi116 to 4016M 2024-04-26T04:53:56.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:56 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:56.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:56 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:56.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:56 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:56.861 INFO:teuthology.orchestra.run.smithi012.stderr:pool 'rbd' created 2024-04-26T04:53:57.490 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:57 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1122210701' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-26T04:53:57.490 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:57 smithi012 ceph-mon[26617]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:57.490 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:57 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:57.490 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:57 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:57.490 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:57 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:57.490 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:57 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:57.491 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:57 smithi012 ceph-mon[26617]: Updating smithi012:/etc/ceph/ceph.conf 2024-04-26T04:53:57.491 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:57 smithi012 ceph-mon[26617]: Updating smithi116:/etc/ceph/ceph.conf 2024-04-26T04:53:57.491 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:57 smithi012 ceph-mon[26617]: mgrmap e20: y(active, since 4s), standbys: x 2024-04-26T04:53:57.491 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:57 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1122210701' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-04-26T04:53:57.491 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:57 smithi012 ceph-mon[26617]: osdmap e61: 8 total, 8 up, 8 in 2024-04-26T04:53:57.491 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:57 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/1122210701' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-26T04:53:57.491 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:57 smithi012 ceph-mon[37567]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:57.491 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:57 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:57.491 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:57 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:57.491 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:57 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:57.491 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:57 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:57.491 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:57 smithi012 ceph-mon[37567]: Updating smithi012:/etc/ceph/ceph.conf 2024-04-26T04:53:57.492 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:57 smithi012 ceph-mon[37567]: Updating smithi116:/etc/ceph/ceph.conf 2024-04-26T04:53:57.492 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:57 smithi012 ceph-mon[37567]: mgrmap e20: y(active, since 4s), standbys: x 2024-04-26T04:53:57.492 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:57 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/1122210701' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-04-26T04:53:57.492 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:57 smithi012 ceph-mon[37567]: osdmap e61: 8 total, 8 up, 8 in 2024-04-26T04:53:57.724 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2024-04-26T04:53:57.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:57 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/1122210701' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-26T04:53:57.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:57 smithi116 ceph-mon[29897]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:57.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:57 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:57.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:57 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:57.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:57 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:57.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:57 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:57.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:57 smithi116 ceph-mon[29897]: Updating smithi012:/etc/ceph/ceph.conf 2024-04-26T04:53:57.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:57 smithi116 ceph-mon[29897]: Updating smithi116:/etc/ceph/ceph.conf 2024-04-26T04:53:57.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:57 smithi116 ceph-mon[29897]: mgrmap e20: y(active, since 4s), standbys: x 2024-04-26T04:53:57.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:57 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/1122210701' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-04-26T04:53:57.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:57 smithi116 ceph-mon[29897]: osdmap e61: 8 total, 8 up, 8 in 2024-04-26T04:53:58.000 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:53:58.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:58 smithi012 ceph-mon[26617]: Updating smithi012:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:53:58.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:58 smithi012 ceph-mon[26617]: Updating smithi116:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:53:58.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:58 smithi012 ceph-mon[26617]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:53:58.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:58 smithi012 ceph-mon[26617]: Updating smithi116:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:53:58.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:58 smithi012 ceph-mon[26617]: Updating smithi116:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:53:58.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:58 smithi012 ceph-mon[26617]: Updating smithi012:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:53:58.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:58 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:58.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:58 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:58.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:58 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:58.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:58 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:58.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:58 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:58.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:58 smithi012 ceph-mon[26617]: Deploying daemon alertmanager.a on smithi012 2024-04-26T04:53:58.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:58 smithi012 ceph-mon[26617]: osdmap e62: 8 total, 8 up, 8 in 2024-04-26T04:53:58.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:58 smithi012 ceph-mon[37567]: Updating smithi012:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:53:58.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:58 smithi012 ceph-mon[37567]: Updating smithi116:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:53:58.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:58 smithi012 ceph-mon[37567]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:53:58.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:58 smithi012 ceph-mon[37567]: Updating smithi116:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:53:58.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:58 smithi012 ceph-mon[37567]: Updating smithi116:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:53:58.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:58 smithi012 ceph-mon[37567]: Updating smithi012:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:53:58.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:58 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:58.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:58 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:58.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:58 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:58.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:58 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:58.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:58 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:58.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:58 smithi012 ceph-mon[37567]: Deploying daemon alertmanager.a on smithi012 2024-04-26T04:53:58.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:58 smithi012 ceph-mon[37567]: osdmap e62: 8 total, 8 up, 8 in 2024-04-26T04:53:58.921 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:58 smithi116 ceph-mon[29897]: Updating smithi012:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:53:58.921 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:58 smithi116 ceph-mon[29897]: Updating smithi116:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:53:58.921 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:58 smithi116 ceph-mon[29897]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:53:58.921 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:58 smithi116 ceph-mon[29897]: Updating smithi116:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:53:58.921 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:58 smithi116 ceph-mon[29897]: Updating smithi116:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:53:58.921 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:58 smithi116 ceph-mon[29897]: Updating smithi012:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:53:58.921 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:58 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:58.921 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:58 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:58.922 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:58 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:58.922 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:58 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:58.922 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:58 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:53:58.922 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:58 smithi116 ceph-mon[29897]: Deploying daemon alertmanager.a on smithi012 2024-04-26T04:53:58.922 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:58 smithi116 ceph-mon[29897]: osdmap e62: 8 total, 8 up, 8 in 2024-04-26T04:53:59.762 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:59 smithi012 ceph-mon[37567]: pgmap v8: 9 pgs: 6 unknown, 3 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:59.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:59 smithi012 ceph-mon[37567]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:53:59.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:53:59 smithi012 ceph-mon[37567]: osdmap e63: 8 total, 8 up, 8 in 2024-04-26T04:53:59.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:59 smithi012 ceph-mon[26617]: pgmap v8: 9 pgs: 6 unknown, 3 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:59.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:59 smithi012 ceph-mon[26617]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:53:59.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:53:59 smithi012 ceph-mon[26617]: osdmap e63: 8 total, 8 up, 8 in 2024-04-26T04:53:59.833 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:59 smithi116 ceph-mon[29897]: pgmap v8: 9 pgs: 6 unknown, 3 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:53:59.833 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:59 smithi116 ceph-mon[29897]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:53:59.833 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:53:59 smithi116 ceph-mon[29897]: osdmap e63: 8 total, 8 up, 8 in 2024-04-26T04:54:00.521 INFO:teuthology.orchestra.run.smithi012.stderr:enabled application 'rbd' on pool 'rbd' 2024-04-26T04:54:00.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:00 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/3259095563' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:54:00.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:00 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/3259095563' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:54:00.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:00 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/3259095563' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:54:01.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:01 smithi012 ceph-mon[26617]: pgmap v10: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-04-26T04:54:01.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:01 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/3259095563' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:54:01.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:01 smithi012 ceph-mon[26617]: osdmap e64: 8 total, 8 up, 8 in 2024-04-26T04:54:01.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:01 smithi012 ceph-mon[37567]: pgmap v10: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-04-26T04:54:01.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:01 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/3259095563' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:54:01.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:01 smithi012 ceph-mon[37567]: osdmap e64: 8 total, 8 up, 8 in 2024-04-26T04:54:01.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:01 smithi116 ceph-mon[29897]: pgmap v10: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-04-26T04:54:01.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:01 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/3259095563' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:54:01.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:01 smithi116 ceph-mon[29897]: osdmap e64: 8 total, 8 up, 8 in 2024-04-26T04:54:02.115 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-26T04:54:02.115 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-26T04:54:02.115 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph mgr dump --format=json 2024-04-26T04:54:02.931 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:02 smithi012 ceph-mon[26617]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:54:02.931 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:02 smithi012 ceph-mon[26617]: Cluster is now healthy 2024-04-26T04:54:02.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:02 smithi012 ceph-mon[37567]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:54:02.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:02 smithi012 ceph-mon[37567]: Cluster is now healthy 2024-04-26T04:54:02.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:02 smithi116 ceph-mon[29897]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:54:02.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:02 smithi116 ceph-mon[29897]: Cluster is now healthy 2024-04-26T04:54:03.416 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:03.850 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:03 smithi012 ceph-mon[26617]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 20 op/s 2024-04-26T04:54:03.850 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:03 smithi012 ceph-mon[37567]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 20 op/s 2024-04-26T04:54:03.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:03 smithi116 ceph-mon[29897]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 20 op/s 2024-04-26T04:54:03.991 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:54:03 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:54:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:54:05.862 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:05 smithi012 ceph-mon[26617]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-26T04:54:05.862 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:05 smithi012 ceph-mon[37567]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-26T04:54:05.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:05 smithi116 ceph-mon[29897]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-26T04:54:06.513 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:54:06 smithi012 systemd[1]: Starting Ceph alertmanager.a for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:54:07.263 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:54:06 smithi012 podman[66265]: 2024-04-26 04:54:06.892261396 +0000 UTC m=+0.031952564 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-04-26T04:54:07.263 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:54:07 smithi012 podman[66265]: 2024-04-26 04:54:07.003164994 +0000 UTC m=+0.142856220 volume create fa8621803054bc2b14da4a9624a776ff9fe95520e05c452d52e5f167fc68f30f 2024-04-26T04:54:07.263 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:54:07 smithi012 podman[66265]: 2024-04-26 04:54:07.111944697 +0000 UTC m=+0.251635896 container create 1a5d42cbab7d497cd40de05c51cf02089025b4b199f8753abbf92a360cfac805 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:54:07.576 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:54:07 smithi012 podman[66265]: 2024-04-26 04:54:07.422534604 +0000 UTC m=+0.562225757 container init 1a5d42cbab7d497cd40de05c51cf02089025b4b199f8753abbf92a360cfac805 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:54:07.576 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:54:07 smithi012 podman[66265]: 2024-04-26 04:54:07.425137297 +0000 UTC m=+0.564828459 container start 1a5d42cbab7d497cd40de05c51cf02089025b4b199f8753abbf92a360cfac805 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:54:07.576 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:54:07 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[66283]: ts=2024-04-26T04:54:07.437Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-04-26T04:54:07.576 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:54:07 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[66283]: ts=2024-04-26T04:54:07.437Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-04-26T04:54:07.576 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:54:07 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[66283]: ts=2024-04-26T04:54:07.438Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.12 port=9094 2024-04-26T04:54:07.576 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:54:07 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[66283]: ts=2024-04-26T04:54:07.439Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-26T04:54:07.576 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:54:07 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[66283]: ts=2024-04-26T04:54:07.462Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-26T04:54:07.576 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:54:07 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[66283]: ts=2024-04-26T04:54:07.463Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-26T04:54:07.576 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:54:07 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[66283]: ts=2024-04-26T04:54:07.466Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-04-26T04:54:07.576 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:54:07 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[66283]: ts=2024-04-26T04:54:07.466Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-04-26T04:54:07.577 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:54:07 smithi012 bash[66265]: 1a5d42cbab7d497cd40de05c51cf02089025b4b199f8753abbf92a360cfac805 2024-04-26T04:54:07.577 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:54:07 smithi012 systemd[1]: Started Ceph alertmanager.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:54:07.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:07 smithi116 ceph-mon[29897]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 14 op/s 2024-04-26T04:54:07.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:07 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:08.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:07 smithi012 ceph-mon[26617]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 14 op/s 2024-04-26T04:54:08.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:07 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:08.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:07 smithi012 ceph-mon[37567]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 14 op/s 2024-04-26T04:54:08.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:07 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:08.013 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:54:07 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: [26/Apr/2024:04:54:07] ENGINE Bus STOPPING 2024-04-26T04:54:08.013 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:54:07 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: [26/Apr/2024:04:54:07] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:54:08.014 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:54:07 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: [26/Apr/2024:04:54:07] ENGINE Bus STOPPED 2024-04-26T04:54:08.014 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:54:07 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: [26/Apr/2024:04:54:07] ENGINE Bus STARTING 2024-04-26T04:54:08.014 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:54:07 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: [26/Apr/2024:04:54:07] ENGINE Serving on http://:::9283 2024-04-26T04:54:08.014 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:54:07 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: [26/Apr/2024:04:54:07] ENGINE Bus STARTED 2024-04-26T04:54:08.020 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:54:08.874 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:08 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:08.874 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:08 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:08.874 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:08 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:08.874 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:08 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:08.874 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:08 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:08.874 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:08 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:08.874 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:08 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T04:54:08.875 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:08 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:08.875 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:08 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/2587856544' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T04:54:08.875 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:08 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:08.875 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:08 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:08.875 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:08 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:08.875 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:08 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:08.875 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:08 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:08.875 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:08 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:08.875 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:08 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T04:54:08.875 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:08 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:08.876 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:08 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/2587856544' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T04:54:08.905 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":20,"flags":0,"active_gid":14454,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6800","nonce":2470533793},{"type":"v1","addr":"172.21.15.12:6801","nonce":2470533793}]},"active_addr":"172.21.15.12:6801/2470533793","active_change":"2024-04-26T04:53:52.395055+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24334,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.12:8443/","prometheus":"http://172.21.15.12:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.12:0","nonce":1826681709}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.12:0","nonce":3390053105}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.12:0","nonce":2567163031}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.12:0","nonce":2417119692}]}]} 2024-04-26T04:54:08.909 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-26T04:54:08.910 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-26T04:54:08.910 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T04:54:08.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:08 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:08.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:08 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:08.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:08 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:08.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:08 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:08.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:08 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:08.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:08 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:08.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:08 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T04:54:08.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:08 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:08.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:08 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/2587856544' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T04:54:09.190 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:09.597 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:54:09 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[66283]: ts=2024-04-26T04:54:09.439Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000256754s 2024-04-26T04:54:09.893 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:09 smithi116 ceph-mon[29897]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T04:54:09.893 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:09 smithi116 ceph-mon[29897]: Deploying daemon grafana.a on smithi116 2024-04-26T04:54:09.893 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:09 smithi116 ceph-mon[29897]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:10.012 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:09 smithi012 ceph-mon[37567]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T04:54:10.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:09 smithi012 ceph-mon[37567]: Deploying daemon grafana.a on smithi116 2024-04-26T04:54:10.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:09 smithi012 ceph-mon[37567]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:10.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:09 smithi012 ceph-mon[26617]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T04:54:10.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:09 smithi012 ceph-mon[26617]: Deploying daemon grafana.a on smithi116 2024-04-26T04:54:10.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:09 smithi012 ceph-mon[26617]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:11.051 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:54:11.052 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":64,"fsid":"0b9350c6-0388-11ef-bc93-c7b262605968","created":"2024-04-26T04:48:17.966774+0000","modified":"2024-04-26T04:54:00.509023+0000","last_up_change":"2024-04-26T04:53:21.501626+0000","last_in_change":"2024-04-26T04:53:04.660871+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:51:27.066997+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-26T04:53:56.453197+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"5f43d5c7-23d9-48a5-bc3c-a3a6ce3c004f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6802","nonce":1077619568},{"type":"v1","addr":"172.21.15.12:6803","nonce":1077619568}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6804","nonce":1077619568},{"type":"v1","addr":"172.21.15.12:6805","nonce":1077619568}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6808","nonce":1077619568},{"type":"v1","addr":"172.21.15.12:6809","nonce":1077619568}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6806","nonce":1077619568},{"type":"v1","addr":"172.21.15.12:6807","nonce":1077619568}]},"public_addr":"172.21.15.12:6803/1077619568","cluster_addr":"172.21.15.12:6805/1077619568","heartbeat_back_addr":"172.21.15.12:6809/1077619568","heartbeat_front_addr":"172.21.15.12:6807/1077619568","state":["exists","up"]},{"osd":1,"uuid":"75471714-6e29-4e02-9c03-86ce919b1ff0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6810","nonce":726116340},{"type":"v1","addr":"172.21.15.12:6811","nonce":726116340}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6812","nonce":726116340},{"type":"v1","addr":"172.21.15.12:6813","nonce":726116340}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6816","nonce":726116340},{"type":"v1","addr":"172.21.15.12:6817","nonce":726116340}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6814","nonce":726116340},{"type":"v1","addr":"172.21.15.12:6815","nonce":726116340}]},"public_addr":"172.21.15.12:6811/726116340","cluster_addr":"172.21.15.12:6813/726116340","heartbeat_back_addr":"172.21.15.12:6817/726116340","heartbeat_front_addr":"172.21.15.12:6815/726116340","state":["exists","up"]},{"osd":2,"uuid":"4b402201-4f28-4bc1-9cf0-0384090fb158","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6818","nonce":4118496923},{"type":"v1","addr":"172.21.15.12:6819","nonce":4118496923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6820","nonce":4118496923},{"type":"v1","addr":"172.21.15.12:6821","nonce":4118496923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6824","nonce":4118496923},{"type":"v1","addr":"172.21.15.12:6825","nonce":4118496923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6822","nonce":4118496923},{"type":"v1","addr":"172.21.15.12:6823","nonce":4118496923}]},"public_addr":"172.21.15.12:6819/4118496923","cluster_addr":"172.21.15.12:6821/4118496923","heartbeat_back_addr":"172.21.15.12:6825/4118496923","heartbeat_front_addr":"172.21.15.12:6823/4118496923","state":["exists","up"]},{"osd":3,"uuid":"54bf03b9-b2e6-4230-8dcd-d62af8853c36","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6826","nonce":3374180434},{"type":"v1","addr":"172.21.15.12:6827","nonce":3374180434}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6828","nonce":3374180434},{"type":"v1","addr":"172.21.15.12:6829","nonce":3374180434}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6832","nonce":3374180434},{"type":"v1","addr":"172.21.15.12:6833","nonce":3374180434}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6830","nonce":3374180434},{"type":"v1","addr":"172.21.15.12:6831","nonce":3374180434}]},"public_addr":"172.21.15.12:6827/3374180434","cluster_addr":"172.21.15.12:6829/3374180434","heartbeat_back_addr":"172.21.15.12:6833/3374180434","heartbeat_front_addr":"172.21.15.12:6831/3374180434","state":["exists","up"]},{"osd":4,"uuid":"ea9d3fb5-1877-4971-a8fb-8b5aafa7871a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6800","nonce":2011782592},{"type":"v1","addr":"172.21.15.116:6801","nonce":2011782592}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6802","nonce":2011782592},{"type":"v1","addr":"172.21.15.116:6803","nonce":2011782592}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6806","nonce":2011782592},{"type":"v1","addr":"172.21.15.116:6807","nonce":2011782592}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6804","nonce":2011782592},{"type":"v1","addr":"172.21.15.116:6805","nonce":2011782592}]},"public_addr":"172.21.15.116:6801/2011782592","cluster_addr":"172.21.15.116:6803/2011782592","heartbeat_back_addr":"172.21.15.116:6807/2011782592","heartbeat_front_addr":"172.21.15.116:6805/2011782592","state":["exists","up"]},{"osd":5,"uuid":"59f0530d-37eb-41d8-86d0-580ba9b8e4e2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6808","nonce":4077435109},{"type":"v1","addr":"172.21.15.116:6809","nonce":4077435109}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6810","nonce":4077435109},{"type":"v1","addr":"172.21.15.116:6811","nonce":4077435109}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6814","nonce":4077435109},{"type":"v1","addr":"172.21.15.116:6815","nonce":4077435109}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6812","nonce":4077435109},{"type":"v1","addr":"172.21.15.116:6813","nonce":4077435109}]},"public_addr":"172.21.15.116:6809/4077435109","cluster_addr":"172.21.15.116:6811/4077435109","heartbeat_back_addr":"172.21.15.116:6815/4077435109","heartbeat_front_addr":"172.21.15.116:6813/4077435109","state":["exists","up"]},{"osd":6,"uuid":"62b1edf5-75b9-4ea6-af34-d40584a6c343","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6816","nonce":2560073849},{"type":"v1","addr":"172.21.15.116:6817","nonce":2560073849}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6818","nonce":2560073849},{"type":"v1","addr":"172.21.15.116:6819","nonce":2560073849}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6822","nonce":2560073849},{"type":"v1","addr":"172.21.15.116:6823","nonce":2560073849}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6820","nonce":2560073849},{"type":"v1","addr":"172.21.15.116:6821","nonce":2560073849}]},"public_addr":"172.21.15.116:6817/2560073849","cluster_addr":"172.21.15.116:6819/2560073849","heartbeat_back_addr":"172.21.15.116:6823/2560073849","heartbeat_front_addr":"172.21.15.116:6821/2560073849","state":["exists","up"]},{"osd":7,"uuid":"439d336e-0012-4b61-893f-50f562863965","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6824","nonce":3779511167},{"type":"v1","addr":"172.21.15.116:6825","nonce":3779511167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6826","nonce":3779511167},{"type":"v1","addr":"172.21.15.116:6827","nonce":3779511167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6830","nonce":3779511167},{"type":"v1","addr":"172.21.15.116:6831","nonce":3779511167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6828","nonce":3779511167},{"type":"v1","addr":"172.21.15.116:6829","nonce":3779511167}]},"public_addr":"172.21.15.116:6825/3779511167","cluster_addr":"172.21.15.116:6827/3779511167","heartbeat_back_addr":"172.21.15.116:6831/3779511167","heartbeat_front_addr":"172.21.15.116:6829/3779511167","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:50:34.985771+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:50:58.677899+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:51:23.046413+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:51:48.496821+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:09.812658+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:32.853940+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:56.215123+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:53:19.840326+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.12:0/625629550":"2024-04-27T04:53:52.394847+0000","172.21.15.12:0/824190310":"2024-04-27T04:53:52.394847+0000","172.21.15.12:0/2421619470":"2024-04-27T04:53:52.394847+0000","172.21.15.12:6801/1235528000":"2024-04-27T04:53:52.394847+0000","172.21.15.12:0/4212767587":"2024-04-27T04:49:16.560212+0000","172.21.15.12:0/1150032450":"2024-04-27T04:49:16.560212+0000","172.21.15.12:6801/2681852455":"2024-04-27T04:49:16.560212+0000","172.21.15.12:6800/1235528000":"2024-04-27T04:53:52.394847+0000","172.21.15.12:6801/378740862":"2024-04-27T04:48:52.473898+0000","172.21.15.12:6800/378740862":"2024-04-27T04:48:52.473898+0000","172.21.15.12:0/320935135":"2024-04-27T04:48:52.473898+0000","172.21.15.12:0/4007370348":"2024-04-27T04:49:16.560212+0000","172.21.15.12:0/2016967349":"2024-04-27T04:48:52.473898+0000","172.21.15.12:0/2418603751":"2024-04-27T04:48:52.473898+0000","172.21.15.12:0/1087977851":"2024-04-27T04:53:52.394847+0000","172.21.15.12:6800/2681852455":"2024-04-27T04:49:16.560212+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T04:54:11.621 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:11 smithi012 ceph-mon[26617]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:11.622 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:11 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1315443832' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:54:11.622 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:11 smithi012 ceph-mon[37567]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:11.622 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:11 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/1315443832' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:54:11.650 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-26T04:54:11.650 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T04:54:11.917 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:11.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:11 smithi116 ceph-mon[29897]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:11.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:11 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/1315443832' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:54:13.720 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:13 smithi012 ceph-mon[26617]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:13.720 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:13 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:13.721 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:13 smithi012 ceph-mon[37567]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:13.721 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:13 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:13.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:13 smithi116 ceph-mon[29897]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:13.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:13 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:13.741 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:54:13 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:54:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:54:13.746 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:54:13.747 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":64,"fsid":"0b9350c6-0388-11ef-bc93-c7b262605968","created":"2024-04-26T04:48:17.966774+0000","modified":"2024-04-26T04:54:00.509023+0000","last_up_change":"2024-04-26T04:53:21.501626+0000","last_in_change":"2024-04-26T04:53:04.660871+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:51:27.066997+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-26T04:53:56.453197+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"5f43d5c7-23d9-48a5-bc3c-a3a6ce3c004f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6802","nonce":1077619568},{"type":"v1","addr":"172.21.15.12:6803","nonce":1077619568}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6804","nonce":1077619568},{"type":"v1","addr":"172.21.15.12:6805","nonce":1077619568}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6808","nonce":1077619568},{"type":"v1","addr":"172.21.15.12:6809","nonce":1077619568}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6806","nonce":1077619568},{"type":"v1","addr":"172.21.15.12:6807","nonce":1077619568}]},"public_addr":"172.21.15.12:6803/1077619568","cluster_addr":"172.21.15.12:6805/1077619568","heartbeat_back_addr":"172.21.15.12:6809/1077619568","heartbeat_front_addr":"172.21.15.12:6807/1077619568","state":["exists","up"]},{"osd":1,"uuid":"75471714-6e29-4e02-9c03-86ce919b1ff0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6810","nonce":726116340},{"type":"v1","addr":"172.21.15.12:6811","nonce":726116340}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6812","nonce":726116340},{"type":"v1","addr":"172.21.15.12:6813","nonce":726116340}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6816","nonce":726116340},{"type":"v1","addr":"172.21.15.12:6817","nonce":726116340}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6814","nonce":726116340},{"type":"v1","addr":"172.21.15.12:6815","nonce":726116340}]},"public_addr":"172.21.15.12:6811/726116340","cluster_addr":"172.21.15.12:6813/726116340","heartbeat_back_addr":"172.21.15.12:6817/726116340","heartbeat_front_addr":"172.21.15.12:6815/726116340","state":["exists","up"]},{"osd":2,"uuid":"4b402201-4f28-4bc1-9cf0-0384090fb158","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6818","nonce":4118496923},{"type":"v1","addr":"172.21.15.12:6819","nonce":4118496923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6820","nonce":4118496923},{"type":"v1","addr":"172.21.15.12:6821","nonce":4118496923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6824","nonce":4118496923},{"type":"v1","addr":"172.21.15.12:6825","nonce":4118496923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6822","nonce":4118496923},{"type":"v1","addr":"172.21.15.12:6823","nonce":4118496923}]},"public_addr":"172.21.15.12:6819/4118496923","cluster_addr":"172.21.15.12:6821/4118496923","heartbeat_back_addr":"172.21.15.12:6825/4118496923","heartbeat_front_addr":"172.21.15.12:6823/4118496923","state":["exists","up"]},{"osd":3,"uuid":"54bf03b9-b2e6-4230-8dcd-d62af8853c36","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6826","nonce":3374180434},{"type":"v1","addr":"172.21.15.12:6827","nonce":3374180434}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6828","nonce":3374180434},{"type":"v1","addr":"172.21.15.12:6829","nonce":3374180434}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6832","nonce":3374180434},{"type":"v1","addr":"172.21.15.12:6833","nonce":3374180434}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6830","nonce":3374180434},{"type":"v1","addr":"172.21.15.12:6831","nonce":3374180434}]},"public_addr":"172.21.15.12:6827/3374180434","cluster_addr":"172.21.15.12:6829/3374180434","heartbeat_back_addr":"172.21.15.12:6833/3374180434","heartbeat_front_addr":"172.21.15.12:6831/3374180434","state":["exists","up"]},{"osd":4,"uuid":"ea9d3fb5-1877-4971-a8fb-8b5aafa7871a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6800","nonce":2011782592},{"type":"v1","addr":"172.21.15.116:6801","nonce":2011782592}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6802","nonce":2011782592},{"type":"v1","addr":"172.21.15.116:6803","nonce":2011782592}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6806","nonce":2011782592},{"type":"v1","addr":"172.21.15.116:6807","nonce":2011782592}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6804","nonce":2011782592},{"type":"v1","addr":"172.21.15.116:6805","nonce":2011782592}]},"public_addr":"172.21.15.116:6801/2011782592","cluster_addr":"172.21.15.116:6803/2011782592","heartbeat_back_addr":"172.21.15.116:6807/2011782592","heartbeat_front_addr":"172.21.15.116:6805/2011782592","state":["exists","up"]},{"osd":5,"uuid":"59f0530d-37eb-41d8-86d0-580ba9b8e4e2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6808","nonce":4077435109},{"type":"v1","addr":"172.21.15.116:6809","nonce":4077435109}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6810","nonce":4077435109},{"type":"v1","addr":"172.21.15.116:6811","nonce":4077435109}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6814","nonce":4077435109},{"type":"v1","addr":"172.21.15.116:6815","nonce":4077435109}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6812","nonce":4077435109},{"type":"v1","addr":"172.21.15.116:6813","nonce":4077435109}]},"public_addr":"172.21.15.116:6809/4077435109","cluster_addr":"172.21.15.116:6811/4077435109","heartbeat_back_addr":"172.21.15.116:6815/4077435109","heartbeat_front_addr":"172.21.15.116:6813/4077435109","state":["exists","up"]},{"osd":6,"uuid":"62b1edf5-75b9-4ea6-af34-d40584a6c343","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6816","nonce":2560073849},{"type":"v1","addr":"172.21.15.116:6817","nonce":2560073849}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6818","nonce":2560073849},{"type":"v1","addr":"172.21.15.116:6819","nonce":2560073849}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6822","nonce":2560073849},{"type":"v1","addr":"172.21.15.116:6823","nonce":2560073849}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6820","nonce":2560073849},{"type":"v1","addr":"172.21.15.116:6821","nonce":2560073849}]},"public_addr":"172.21.15.116:6817/2560073849","cluster_addr":"172.21.15.116:6819/2560073849","heartbeat_back_addr":"172.21.15.116:6823/2560073849","heartbeat_front_addr":"172.21.15.116:6821/2560073849","state":["exists","up"]},{"osd":7,"uuid":"439d336e-0012-4b61-893f-50f562863965","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6824","nonce":3779511167},{"type":"v1","addr":"172.21.15.116:6825","nonce":3779511167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6826","nonce":3779511167},{"type":"v1","addr":"172.21.15.116:6827","nonce":3779511167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6830","nonce":3779511167},{"type":"v1","addr":"172.21.15.116:6831","nonce":3779511167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6828","nonce":3779511167},{"type":"v1","addr":"172.21.15.116:6829","nonce":3779511167}]},"public_addr":"172.21.15.116:6825/3779511167","cluster_addr":"172.21.15.116:6827/3779511167","heartbeat_back_addr":"172.21.15.116:6831/3779511167","heartbeat_front_addr":"172.21.15.116:6829/3779511167","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:50:34.985771+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:50:58.677899+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:51:23.046413+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:51:48.496821+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:09.812658+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:32.853940+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:56.215123+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:53:19.840326+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.12:0/625629550":"2024-04-27T04:53:52.394847+0000","172.21.15.12:0/824190310":"2024-04-27T04:53:52.394847+0000","172.21.15.12:0/2421619470":"2024-04-27T04:53:52.394847+0000","172.21.15.12:6801/1235528000":"2024-04-27T04:53:52.394847+0000","172.21.15.12:0/4212767587":"2024-04-27T04:49:16.560212+0000","172.21.15.12:0/1150032450":"2024-04-27T04:49:16.560212+0000","172.21.15.12:6801/2681852455":"2024-04-27T04:49:16.560212+0000","172.21.15.12:6800/1235528000":"2024-04-27T04:53:52.394847+0000","172.21.15.12:6801/378740862":"2024-04-27T04:48:52.473898+0000","172.21.15.12:6800/378740862":"2024-04-27T04:48:52.473898+0000","172.21.15.12:0/320935135":"2024-04-27T04:48:52.473898+0000","172.21.15.12:0/4007370348":"2024-04-27T04:49:16.560212+0000","172.21.15.12:0/2016967349":"2024-04-27T04:48:52.473898+0000","172.21.15.12:0/2418603751":"2024-04-27T04:48:52.473898+0000","172.21.15.12:0/1087977851":"2024-04-27T04:53:52.394847+0000","172.21.15.12:6800/2681852455":"2024-04-27T04:49:16.560212+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T04:54:14.704 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-26T04:54:14.704 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-26T04:54:14.704 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-26T04:54:14.705 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-26T04:54:14.705 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-26T04:54:14.706 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-26T04:54:14.706 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-26T04:54:14.707 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-26T04:54:14.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:14 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/3999649496' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:54:14.741 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:14 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/3999649496' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:54:14.742 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:14 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/3999649496' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:54:15.230 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:15.248 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:15.249 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:15.253 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:15.257 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:15.260 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:15.260 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:15.269 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:15.671 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:15 smithi012 ceph-mon[26617]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:15.671 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:15 smithi012 ceph-mon[37567]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:15.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:15 smithi116 ceph-mon[29897]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:17.739 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:17 smithi012 ceph-mon[26617]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:17.739 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:17 smithi012 ceph-mon[37567]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:17.739 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:54:17 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[66283]: ts=2024-04-26T04:54:17.441Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002052544s 2024-04-26T04:54:17.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:17 smithi116 ceph-mon[29897]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:19.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:19 smithi012 ceph-mon[26617]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:19.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:19 smithi012 ceph-mon[37567]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:19.594 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:19 smithi116 ceph-mon[29897]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:20.992 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:20 smithi116 systemd[1]: Starting Ceph grafana.a for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:54:21.306 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 podman[59094]: 2024-04-26 04:54:20.926087041 +0000 UTC m=+0.017523648 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:54:21.306 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 podman[59094]: 2024-04-26 04:54:21.06211251 +0000 UTC m=+0.153549092 container create c7fd7ca82930f4afcf7e2c48112817b9cdfcc301586b96887e7e6f19b48111c6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, io.buildah.version=1.32.0, version=8.5, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, io.openshift.tags=base rhel8, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, release=239.1651231664, name=ubi8) 2024-04-26T04:54:21.306 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 podman[59094]: 2024-04-26 04:54:21.306052309 +0000 UTC m=+0.397488911 container init c7fd7ca82930f4afcf7e2c48112817b9cdfcc301586b96887e7e6f19b48111c6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, io.buildah.version=1.32.0, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, release=239.1651231664, distribution-scope=public, io.openshift.tags=base rhel8, name=ubi8, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, vcs-type=git) 2024-04-26T04:54:21.557 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-mon[29897]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:21.557 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:21.557 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:21.558 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 podman[59094]: 2024-04-26 04:54:21.308598747 +0000 UTC m=+0.400035333 container start c7fd7ca82930f4afcf7e2c48112817b9cdfcc301586b96887e7e6f19b48111c6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64, description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, vcs-type=git, release=239.1651231664, distribution-scope=public, io.openshift.tags=base rhel8, version=8.5, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897) 2024-04-26T04:54:21.558 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 bash[59094]: c7fd7ca82930f4afcf7e2c48112817b9cdfcc301586b96887e7e6f19b48111c6 2024-04-26T04:54:21.558 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 systemd[1]: Started Ceph grafana.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:54:21.558 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=settings t=2024-04-26T04:54:21.465983365Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-04-26T04:54:21.558 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=settings t=2024-04-26T04:54:21.466167288Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-04-26T04:54:21.558 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=settings t=2024-04-26T04:54:21.46618431Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-04-26T04:54:21.559 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=settings t=2024-04-26T04:54:21.46619052Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-04-26T04:54:21.559 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=settings t=2024-04-26T04:54:21.466196478Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-04-26T04:54:21.559 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=settings t=2024-04-26T04:54:21.46620201Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-04-26T04:54:21.559 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=settings t=2024-04-26T04:54:21.466207283Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-04-26T04:54:21.559 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=settings t=2024-04-26T04:54:21.466212758Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-04-26T04:54:21.559 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=settings t=2024-04-26T04:54:21.466218678Z level=info msg="Path Home" path=/usr/share/grafana 2024-04-26T04:54:21.559 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=settings t=2024-04-26T04:54:21.466223927Z level=info msg="Path Data" path=/var/lib/grafana 2024-04-26T04:54:21.559 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=settings t=2024-04-26T04:54:21.466229068Z level=info msg="Path Logs" path=/var/log/grafana 2024-04-26T04:54:21.559 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=settings t=2024-04-26T04:54:21.466234101Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-04-26T04:54:21.559 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=settings t=2024-04-26T04:54:21.466239229Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-04-26T04:54:21.559 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=settings t=2024-04-26T04:54:21.466244372Z level=info msg="App mode production" 2024-04-26T04:54:21.559 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=sqlstore t=2024-04-26T04:54:21.466291406Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-04-26T04:54:21.559 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=sqlstore t=2024-04-26T04:54:21.466306707Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-04-26T04:54:21.560 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.467517492Z level=info msg="Starting DB migrations" 2024-04-26T04:54:21.560 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.467621392Z level=info msg="Executing migration" id="create migration_log table" 2024-04-26T04:54:21.560 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.468158737Z level=info msg="Executing migration" id="create user table" 2024-04-26T04:54:21.560 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.468618391Z level=info msg="Executing migration" id="add unique index user.login" 2024-04-26T04:54:21.560 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.469100855Z level=info msg="Executing migration" id="add unique index user.email" 2024-04-26T04:54:21.560 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.469542546Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-04-26T04:54:21.560 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.470137334Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-04-26T04:54:21.560 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.470534882Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-04-26T04:54:21.560 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.471245645Z level=info msg="Executing migration" id="create user table v2" 2024-04-26T04:54:21.560 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.473543487Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-04-26T04:54:21.560 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.47621286Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-04-26T04:54:21.560 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.479047905Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-04-26T04:54:21.560 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.48172491Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-04-26T04:54:21.560 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.484485444Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-04-26T04:54:21.561 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.487169702Z level=info msg="Executing migration" id="Update user table charset" 2024-04-26T04:54:21.561 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.489739696Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-04-26T04:54:21.561 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.490201843Z level=info msg="Executing migration" id="Add missing user data" 2024-04-26T04:54:21.561 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.493045997Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-04-26T04:54:21.561 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.495709884Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-04-26T04:54:21.561 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.498486542Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-04-26T04:54:21.561 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.501433701Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-04-26T04:54:21.561 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.504024002Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-04-26T04:54:21.561 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.504521173Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-04-26T04:54:21.561 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.507417228Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-04-26T04:54:21.561 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.510185807Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-04-26T04:54:21.561 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.512935276Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-04-26T04:54:21.562 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.515410412Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-04-26T04:54:21.562 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.517976984Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-04-26T04:54:21.562 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.520650076Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-04-26T04:54:21.562 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.523249524Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-04-26T04:54:21.562 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.525977613Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-04-26T04:54:21.562 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.528359547Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-04-26T04:54:21.562 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.53102364Z level=info msg="Executing migration" id="create temp_user v2" 2024-04-26T04:54:21.562 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.533847087Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-04-26T04:54:21.562 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.536272411Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-04-26T04:54:21.562 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.538960688Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-04-26T04:54:21.562 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.541435669Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-04-26T04:54:21.562 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.544349848Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-04-26T04:54:21.562 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.546818568Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-04-26T04:54:21.563 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.549528893Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-04-26T04:54:21.563 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.552052454Z level=info msg="Executing migration" id="create star table" 2024-04-26T04:54:21.563 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.552437485Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-04-26T04:54:21.563 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.555271204Z level=info msg="Executing migration" id="create org table v1" 2024-04-26T04:54:21.563 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.555972132Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-04-26T04:54:21.563 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.556966352Z level=info msg="Executing migration" id="create org_user table v1" 2024-04-26T04:54:21.563 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.557669665Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-04-26T04:54:21.563 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.558487314Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-04-26T04:54:21.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:21 smithi012 ceph-mon[26617]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:21.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:21 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:21.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:21 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:21.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:21 smithi012 ceph-mon[37567]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:21.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:21 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:21.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:21 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:21.810 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.559356126Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-04-26T04:54:21.810 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.559889254Z level=info msg="Executing migration" id="Update org table charset" 2024-04-26T04:54:21.810 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.560659654Z level=info msg="Executing migration" id="Update org_user table charset" 2024-04-26T04:54:21.811 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.561503103Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-04-26T04:54:21.811 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.56236215Z level=info msg="Executing migration" id="create dashboard table" 2024-04-26T04:54:21.811 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.562861072Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-04-26T04:54:21.811 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.563694931Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-04-26T04:54:21.811 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.564531624Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-04-26T04:54:21.811 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.565352231Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-04-26T04:54:21.811 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.565838714Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-04-26T04:54:21.811 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.566673615Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-04-26T04:54:21.811 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.567517001Z level=info msg="Executing migration" id="create dashboard v2" 2024-04-26T04:54:21.811 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.568449883Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-04-26T04:54:21.811 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.569046488Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-04-26T04:54:21.812 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.570009423Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-04-26T04:54:21.812 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.570710734Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-04-26T04:54:21.812 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.571700221Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-04-26T04:54:21.812 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.572409046Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-04-26T04:54:21.812 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.573295254Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-04-26T04:54:21.812 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.573954752Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-04-26T04:54:21.812 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.574460112Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-04-26T04:54:21.812 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.575370302Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-04-26T04:54:21.812 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.57604253Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-04-26T04:54:21.812 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.576458213Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-04-26T04:54:21.812 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.577257857Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-04-26T04:54:21.813 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.577558576Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-04-26T04:54:21.813 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.578102064Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-04-26T04:54:21.813 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.578632643Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-04-26T04:54:21.813 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.579118513Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-04-26T04:54:21.813 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.57959682Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-04-26T04:54:21.813 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.580466727Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-04-26T04:54:21.813 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.583206948Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-04-26T04:54:21.813 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.586150035Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-04-26T04:54:21.813 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.586584944Z level=info msg="Executing migration" id="Update dashboard title length" 2024-04-26T04:54:21.813 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.589235406Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-04-26T04:54:21.814 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.58967656Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-04-26T04:54:21.814 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.592007989Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-04-26T04:54:21.814 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.594844413Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-04-26T04:54:21.815 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.595264205Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-04-26T04:54:21.815 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.598127933Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-04-26T04:54:21.815 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.600999175Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-04-26T04:54:21.815 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.603605439Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-04-26T04:54:21.815 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.604101305Z level=info msg="Executing migration" id="Add check_sum column" 2024-04-26T04:54:21.815 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.606994761Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-04-26T04:54:21.816 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.60975436Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-04-26T04:54:21.816 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.612575707Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-04-26T04:54:21.816 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.612904191Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-04-26T04:54:21.816 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.615636838Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-04-26T04:54:21.816 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.618583583Z level=info msg="Executing migration" id="create data_source table" 2024-04-26T04:54:21.816 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.621085709Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-04-26T04:54:21.816 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.621489187Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-04-26T04:54:21.816 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.624287746Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-04-26T04:54:21.816 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.626834771Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-04-26T04:54:21.816 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.629562558Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-04-26T04:54:21.816 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.632276248Z level=info msg="Executing migration" id="create data_source table v2" 2024-04-26T04:54:21.816 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.634610137Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-04-26T04:54:21.816 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.637183928Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-04-26T04:54:21.816 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.639883868Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-04-26T04:54:21.817 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.642346158Z level=info msg="Executing migration" id="Add column with_credentials" 2024-04-26T04:54:21.817 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.645054274Z level=info msg="Executing migration" id="Add secure json data column" 2024-04-26T04:54:21.817 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.64770701Z level=info msg="Executing migration" id="Update data_source table charset" 2024-04-26T04:54:21.817 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.650333075Z level=info msg="Executing migration" id="Update initial version to 1" 2024-04-26T04:54:21.817 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.653041304Z level=info msg="Executing migration" id="Add read_only data column" 2024-04-26T04:54:21.817 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.655491648Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-04-26T04:54:21.817 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.658100189Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-04-26T04:54:21.817 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.660676018Z level=info msg="Executing migration" id="Add uid column" 2024-04-26T04:54:21.817 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.661539902Z level=info msg="Executing migration" id="Update uid value" 2024-04-26T04:54:21.817 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.662363168Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-04-26T04:54:21.817 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.663226425Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-04-26T04:54:21.817 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.663975671Z level=info msg="Executing migration" id="create api_key table" 2024-04-26T04:54:21.817 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.66482803Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-04-26T04:54:21.817 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.665762687Z level=info msg="Executing migration" id="add index api_key.key" 2024-04-26T04:54:21.818 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.666526354Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-04-26T04:54:21.818 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.667366784Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-04-26T04:54:21.818 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.668204651Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-04-26T04:54:21.818 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.668986741Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-04-26T04:54:21.818 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.66980116Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-04-26T04:54:21.818 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.670835151Z level=info msg="Executing migration" id="create api_key table v2" 2024-04-26T04:54:21.818 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.671680051Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-04-26T04:54:21.818 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.672542674Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-04-26T04:54:21.818 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.673489714Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-04-26T04:54:21.818 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.674122859Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-04-26T04:54:21.818 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.676990295Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-04-26T04:54:21.818 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.677358435Z level=info msg="Executing migration" id="Update api_key table charset" 2024-04-26T04:54:21.818 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.680147834Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-04-26T04:54:21.818 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.680830288Z level=info msg="Executing migration" id="Add service account foreign key" 2024-04-26T04:54:21.819 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.68139487Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-04-26T04:54:21.819 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.68420554Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-04-26T04:54:21.819 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.684801823Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-04-26T04:54:21.819 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.687159046Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-04-26T04:54:21.819 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.689687545Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-04-26T04:54:21.819 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.690092403Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-04-26T04:54:21.819 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.693014831Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-04-26T04:54:21.819 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.695543938Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-04-26T04:54:21.819 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.698395417Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-04-26T04:54:21.819 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.698963765Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-04-26T04:54:21.819 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.701551883Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-04-26T04:54:21.819 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.704104065Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-04-26T04:54:21.819 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.706789931Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-04-26T04:54:21.819 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.707446863Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-04-26T04:54:21.820 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.710297858Z level=info msg="Executing migration" id="create quota table v1" 2024-04-26T04:54:21.820 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.71070773Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-04-26T04:54:21.820 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.712884697Z level=info msg="Executing migration" id="Update quota table charset" 2024-04-26T04:54:21.820 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.713229606Z level=info msg="Executing migration" id="create plugin_setting table" 2024-04-26T04:54:21.820 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.715689405Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-04-26T04:54:21.820 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.718435387Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-04-26T04:54:21.820 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.720973081Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-04-26T04:54:21.820 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.72129863Z level=info msg="Executing migration" id="create session table" 2024-04-26T04:54:21.820 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.723621501Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-04-26T04:54:21.820 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.726292152Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-04-26T04:54:21.820 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.726891348Z level=info msg="Executing migration" id="create playlist table v2" 2024-04-26T04:54:21.820 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.727331895Z level=info msg="Executing migration" id="create playlist item table v2" 2024-04-26T04:54:21.820 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.727742158Z level=info msg="Executing migration" id="Update playlist table charset" 2024-04-26T04:54:21.820 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.728458755Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-04-26T04:54:21.821 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.72879249Z level=info msg="Executing migration" id="drop preferences table v2" 2024-04-26T04:54:21.821 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.729125013Z level=info msg="Executing migration" id="drop preferences table v3" 2024-04-26T04:54:21.821 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.729465368Z level=info msg="Executing migration" id="create preferences table v3" 2024-04-26T04:54:21.821 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.729900946Z level=info msg="Executing migration" id="Update preferences table charset" 2024-04-26T04:54:21.821 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.730636231Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-04-26T04:54:21.821 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.731307153Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-04-26T04:54:21.821 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.731654316Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-04-26T04:54:21.821 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.732300821Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-04-26T04:54:21.821 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.733117879Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-04-26T04:54:21.821 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.733452801Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-04-26T04:54:21.821 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.733910194Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-04-26T04:54:21.821 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.734735629Z level=info msg="Executing migration" id="create alert table v1" 2024-04-26T04:54:21.821 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.735540166Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-04-26T04:54:21.822 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.73641589Z level=info msg="Executing migration" id="add index alert state" 2024-04-26T04:54:21.822 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.736873512Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-04-26T04:54:21.822 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.737321753Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-04-26T04:54:21.822 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.737718134Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-04-26T04:54:21.822 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.738188858Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-04-26T04:54:21.822 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.738618251Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-04-26T04:54:21.822 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.74000164Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-04-26T04:54:21.822 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.740411837Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-04-26T04:54:21.822 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.740860313Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-04-26T04:54:21.822 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.741212116Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-04-26T04:54:21.822 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.741555161Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-04-26T04:54:21.822 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.741914628Z level=info msg="Executing migration" id="Add column is_default" 2024-04-26T04:54:21.822 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.742565946Z level=info msg="Executing migration" id="Add column frequency" 2024-04-26T04:54:21.822 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.743243304Z level=info msg="Executing migration" id="Add column send_reminder" 2024-04-26T04:54:21.823 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.743928314Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-04-26T04:54:21.823 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.744582717Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-04-26T04:54:21.823 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.745042129Z level=info msg="Executing migration" id="Update alert table charset" 2024-04-26T04:54:21.823 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.74535851Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-04-26T04:54:21.823 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.745637912Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-04-26T04:54:21.823 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.746077241Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-04-26T04:54:21.823 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.746540036Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-04-26T04:54:21.823 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.746903091Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-04-26T04:54:21.823 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.747340515Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-04-26T04:54:21.823 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.74773646Z level=info msg="Executing migration" id="Add for to alert table" 2024-04-26T04:54:21.823 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.748402263Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-04-26T04:54:21.823 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.749124734Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-04-26T04:54:21.823 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.7494676Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-04-26T04:54:21.823 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.749923861Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-04-26T04:54:21.824 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.750360346Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-04-26T04:54:21.824 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.751016708Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-04-26T04:54:21.824 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.751342764Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-04-26T04:54:21.824 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.751739878Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-04-26T04:54:21.824 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.75216243Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-04-26T04:54:21.824 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.752490353Z level=info msg="Executing migration" id="create annotation table v5" 2024-04-26T04:54:21.824 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.752936662Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-04-26T04:54:21.824 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.753398691Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-04-26T04:54:21.824 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.753857556Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-04-26T04:54:21.824 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.75431574Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-04-26T04:54:21.824 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.754981121Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-04-26T04:54:21.824 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.755466689Z level=info msg="Executing migration" id="Update annotation table charset" 2024-04-26T04:54:21.824 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.755738245Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-04-26T04:54:21.824 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.756452876Z level=info msg="Executing migration" id="Drop category_id index" 2024-04-26T04:54:21.825 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.757283189Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-04-26T04:54:21.825 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.758182823Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-04-26T04:54:21.825 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.758611581Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-04-26T04:54:21.825 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.759637769Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-04-26T04:54:21.826 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.760217997Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-04-26T04:54:21.826 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.76175888Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-04-26T04:54:21.826 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.762155851Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-04-26T04:54:21.826 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.76301411Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-04-26T04:54:21.826 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.763953137Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-04-26T04:54:21.826 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.764679596Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-04-26T04:54:21.826 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.765011438Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-04-26T04:54:21.826 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.765903482Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-04-26T04:54:21.826 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.766923652Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-04-26T04:54:21.826 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.767665066Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-04-26T04:54:21.826 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.768089251Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-04-26T04:54:21.826 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.76907439Z level=info msg="Executing migration" id="Add epoch_end column" 2024-04-26T04:54:21.826 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.769942526Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-04-26T04:54:21.826 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.770767447Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-04-26T04:54:21.827 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.771684701Z level=info msg="Executing migration" id="Move region to single row" 2024-04-26T04:54:21.827 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.772424759Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-04-26T04:54:21.827 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.773284806Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-04-26T04:54:21.827 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.774246813Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-04-26T04:54:21.827 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.776999745Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-04-26T04:54:21.827 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.779703609Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-04-26T04:54:21.827 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.782499025Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-04-26T04:54:21.827 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.782939061Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-04-26T04:54:21.827 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.785749549Z level=info msg="Executing migration" id="create test_data table" 2024-04-26T04:54:21.828 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.788384204Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-04-26T04:54:21.828 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.788936662Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-04-26T04:54:21.828 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.789352672Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-04-26T04:54:21.828 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.792086868Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-04-26T04:54:21.828 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.794976821Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-04-26T04:54:21.828 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.797711827Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-04-26T04:54:21.828 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.798039576Z level=info msg="Executing migration" id="create team table" 2024-04-26T04:54:21.828 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.800948849Z level=info msg="Executing migration" id="add index team.org_id" 2024-04-26T04:54:21.828 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.803482931Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-04-26T04:54:21.828 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.806338272Z level=info msg="Executing migration" id="create team member table" 2024-04-26T04:54:21.828 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.806751391Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-04-26T04:54:21.828 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.809816566Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-04-26T04:54:21.828 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.812382639Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-04-26T04:54:22.062 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.814987128Z level=info msg="Executing migration" id="Add column email to team table" 2024-04-26T04:54:22.062 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.815704027Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-04-26T04:54:22.062 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.818515928Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-04-26T04:54:22.063 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.821161055Z level=info msg="Executing migration" id="create dashboard acl table" 2024-04-26T04:54:22.063 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.823848028Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-04-26T04:54:22.063 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.824385156Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-04-26T04:54:22.063 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.82708516Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-04-26T04:54:22.063 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.827652447Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-04-26T04:54:22.063 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.830329342Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-04-26T04:54:22.063 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.830913681Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-04-26T04:54:22.063 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.83175713Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-04-26T04:54:22.063 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.832603061Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-04-26T04:54:22.063 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.833456319Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-04-26T04:54:22.063 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.834272301Z level=info msg="Executing migration" id="create tag table" 2024-04-26T04:54:22.064 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.835140141Z level=info msg="Executing migration" id="add index tag.key_value" 2024-04-26T04:54:22.064 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.836103422Z level=info msg="Executing migration" id="create login attempt table" 2024-04-26T04:54:22.064 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.836833283Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-04-26T04:54:22.064 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.837715604Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-04-26T04:54:22.064 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.838608294Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-04-26T04:54:22.065 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.840504793Z level=info msg="Executing migration" id="create login_attempt v2" 2024-04-26T04:54:22.065 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.841429792Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-04-26T04:54:22.065 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.842259629Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-04-26T04:54:22.065 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.843197745Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-04-26T04:54:22.065 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.84394797Z level=info msg="Executing migration" id="create user auth table" 2024-04-26T04:54:22.065 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.844954209Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-04-26T04:54:22.065 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.845654752Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-04-26T04:54:22.065 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.846503004Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-04-26T04:54:22.066 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.847306835Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-04-26T04:54:22.066 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.848059943Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-04-26T04:54:22.066 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.848843801Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-04-26T04:54:22.066 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.849936411Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-04-26T04:54:22.066 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.850592384Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-04-26T04:54:22.066 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.85355089Z level=info msg="Executing migration" id="create server_lock table" 2024-04-26T04:54:22.066 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.856323461Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-04-26T04:54:22.066 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.859024009Z level=info msg="Executing migration" id="create user auth token table" 2024-04-26T04:54:22.066 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.861822536Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-04-26T04:54:22.066 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.864656623Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-04-26T04:54:22.066 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.8675392Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-04-26T04:54:22.066 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.870190569Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-04-26T04:54:22.066 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.872990485Z level=info msg="Executing migration" id="create cache_data table" 2024-04-26T04:54:22.066 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.875720153Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-04-26T04:54:22.066 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.878498647Z level=info msg="Executing migration" id="create short_url table v1" 2024-04-26T04:54:22.067 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.881681738Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-04-26T04:54:22.067 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.884079439Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-04-26T04:54:22.067 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.886940036Z level=info msg="Executing migration" id="delete alert_definition table" 2024-04-26T04:54:22.067 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.889675085Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-04-26T04:54:22.067 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.89221212Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-04-26T04:54:22.067 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.89512443Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-04-26T04:54:22.067 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.897957917Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-04-26T04:54:22.067 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.900417367Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-04-26T04:54:22.067 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.902904993Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-04-26T04:54:22.067 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.905451992Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-04-26T04:54:22.067 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.908010629Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-04-26T04:54:22.067 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.910800671Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-04-26T04:54:22.067 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.913362745Z level=info msg="Executing migration" id="drop alert_definition table" 2024-04-26T04:54:22.067 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.916050125Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-04-26T04:54:22.067 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.918781481Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-04-26T04:54:22.067 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.921527002Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-04-26T04:54:22.068 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.924169631Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-04-26T04:54:22.068 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.926991768Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-04-26T04:54:22.068 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.929399475Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-04-26T04:54:22.068 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.932163445Z level=info msg="Executing migration" id="create alert_instance table" 2024-04-26T04:54:22.068 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.934916051Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-04-26T04:54:22.068 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.937589479Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-04-26T04:54:22.068 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.94019289Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-04-26T04:54:22.068 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.94275923Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-04-26T04:54:22.068 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.945400792Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-04-26T04:54:22.068 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.946373801Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-04-26T04:54:22.068 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.949826377Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-04-26T04:54:22.068 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.953143468Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-04-26T04:54:22.068 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.953996175Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-04-26T04:54:22.068 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.954875862Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-04-26T04:54:22.068 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.955721514Z level=info msg="Executing migration" id="create alert_rule table" 2024-04-26T04:54:22.068 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.956617881Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-04-26T04:54:22.069 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.957275637Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-04-26T04:54:22.069 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.958260506Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-04-26T04:54:22.069 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.95897956Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-04-26T04:54:22.069 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.959977237Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-04-26T04:54:22.069 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.960914306Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-04-26T04:54:22.069 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.961758058Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-04-26T04:54:22.069 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.962573246Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-04-26T04:54:22.069 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.963510081Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-04-26T04:54:22.069 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.964205654Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-04-26T04:54:22.069 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.96507286Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-04-26T04:54:22.069 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.96594859Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-04-26T04:54:22.069 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.966964682Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-04-26T04:54:22.069 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.967831276Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-04-26T04:54:22.069 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.968803781Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-04-26T04:54:22.069 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.96957021Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-04-26T04:54:22.069 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.970548458Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-04-26T04:54:22.070 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.971260141Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-04-26T04:54:22.070 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.972103234Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-04-26T04:54:22.070 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.973080176Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-04-26T04:54:22.070 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.973917128Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-04-26T04:54:22.070 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.974753751Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-04-26T04:54:22.070 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.975599291Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-04-26T04:54:22.070 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.976483839Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-04-26T04:54:22.070 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.977350641Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-04-26T04:54:22.070 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.978237182Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-04-26T04:54:22.070 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.979166973Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-04-26T04:54:22.070 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.98010853Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-04-26T04:54:22.070 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.980976061Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-04-26T04:54:22.070 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.983892524Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-04-26T04:54:22.070 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.9866758Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-04-26T04:54:22.071 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.989528589Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-04-26T04:54:22.071 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.992228578Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-04-26T04:54:22.071 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.995078664Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-04-26T04:54:22.071 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:21.997811239Z level=info msg="Executing migration" id="create provenance_type table" 2024-04-26T04:54:22.071 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.000624804Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-04-26T04:54:22.071 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.003287416Z level=info msg="Executing migration" id="create alert_image table" 2024-04-26T04:54:22.071 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.00629878Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-04-26T04:54:22.071 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.009013006Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-04-26T04:54:22.071 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.011695531Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-04-26T04:54:22.071 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.01444669Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-04-26T04:54:22.071 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.017214788Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-04-26T04:54:22.071 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.017308026Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-04-26T04:54:22.071 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.019945712Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-04-26T04:54:22.071 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.022941313Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-04-26T04:54:22.071 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.025721961Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-04-26T04:54:22.072 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.025848966Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-04-26T04:54:22.072 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.025963674Z level=info msg="alerts found to migrate" alerts=0 2024-04-26T04:54:22.072 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.028353984Z level=info msg="Executing migration" id="create library_element table v1" 2024-04-26T04:54:22.072 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.031329144Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-04-26T04:54:22.072 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.033913771Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-04-26T04:54:22.072 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.036698805Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-04-26T04:54:22.072 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.03929134Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-04-26T04:54:22.072 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.041943157Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-04-26T04:54:22.072 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.044636981Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-04-26T04:54:22.072 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.047141728Z level=info msg="Executing migration" id="create data_keys table" 2024-04-26T04:54:22.072 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.049891815Z level=info msg="Executing migration" id="create secrets table" 2024-04-26T04:54:22.072 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.052606293Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-04-26T04:54:22.072 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.056833672Z level=info msg="Executing migration" id="add name column into data_keys" 2024-04-26T04:54:22.072 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.059665445Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-04-26T04:54:22.073 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.06212973Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-04-26T04:54:22.471 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.069490818Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-04-26T04:54:22.471 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.075847195Z level=info msg="Executing migration" id="create kv_store table v1" 2024-04-26T04:54:22.471 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.078517039Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-04-26T04:54:22.471 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.081065187Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-04-26T04:54:22.471 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.083593087Z level=info msg="Executing migration" id="create permission table" 2024-04-26T04:54:22.471 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.086385298Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-04-26T04:54:22.471 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.087027956Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-04-26T04:54:22.471 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.087724557Z level=info msg="Executing migration" id="create role table" 2024-04-26T04:54:22.471 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.088636831Z level=info msg="Executing migration" id="add column display_name" 2024-04-26T04:54:22.471 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.089630821Z level=info msg="Executing migration" id="add column group_name" 2024-04-26T04:54:22.471 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.090578244Z level=info msg="Executing migration" id="add index role.org_id" 2024-04-26T04:54:22.471 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.091424011Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-04-26T04:54:22.472 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.092357691Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-04-26T04:54:22.472 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.093133756Z level=info msg="Executing migration" id="create team role table" 2024-04-26T04:54:22.472 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.094110362Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-04-26T04:54:22.472 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.09482903Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-04-26T04:54:22.472 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.095641588Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-04-26T04:54:22.472 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.096429017Z level=info msg="Executing migration" id="create user role table" 2024-04-26T04:54:22.472 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.097264696Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-04-26T04:54:22.472 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.098150701Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-04-26T04:54:22.472 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.098975857Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-04-26T04:54:22.472 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.099955351Z level=info msg="Executing migration" id="create builtin role table" 2024-04-26T04:54:22.472 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.100706686Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-04-26T04:54:22.472 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.101459468Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-04-26T04:54:22.472 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.102511576Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-04-26T04:54:22.473 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.103540962Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-04-26T04:54:22.473 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.104026542Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-04-26T04:54:22.473 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.104817189Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-04-26T04:54:22.473 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.105224804Z level=info msg="Executing migration" id="add unique index role.uid" 2024-04-26T04:54:22.473 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.105614672Z level=info msg="Executing migration" id="create seed assignment table" 2024-04-26T04:54:22.473 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.10601706Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-04-26T04:54:22.473 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.106441348Z level=info msg="Executing migration" id="add column hidden to role table" 2024-04-26T04:54:22.473 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.107422981Z level=info msg="Executing migration" id="create query_history table v1" 2024-04-26T04:54:22.473 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.107816133Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-04-26T04:54:22.473 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.10824788Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-04-26T04:54:22.473 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.108509468Z level=info msg="Executing migration" id="teams permissions migration" 2024-04-26T04:54:22.473 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.108891512Z level=info msg="Executing migration" id="dashboard permissions" 2024-04-26T04:54:22.473 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.109364381Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-04-26T04:54:22.474 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.109697149Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-04-26T04:54:22.474 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.110004683Z level=info msg="Executing migration" id="alerting notification permissions" 2024-04-26T04:54:22.474 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.110354366Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-04-26T04:54:22.474 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.110732372Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-04-26T04:54:22.475 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.111159374Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-04-26T04:54:22.475 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.112131638Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-04-26T04:54:22.475 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.11244634Z level=info msg="Executing migration" id="create correlation table v1" 2024-04-26T04:54:22.475 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.112847982Z level=info msg="Executing migration" id="add index correlations.uid" 2024-04-26T04:54:22.475 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.113290671Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-04-26T04:54:22.475 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.113727985Z level=info msg="Executing migration" id="add correlation config column" 2024-04-26T04:54:22.475 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.1146993Z level=info msg="Executing migration" id="create entity_events table" 2024-04-26T04:54:22.475 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.115125368Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-04-26T04:54:22.475 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.115530382Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-04-26T04:54:22.475 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.115607307Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-04-26T04:54:22.475 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.115896178Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-26T04:54:22.475 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.115976961Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-26T04:54:22.475 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.116241309Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-04-26T04:54:22.475 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.116616415Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-04-26T04:54:22.476 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.11703167Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-04-26T04:54:22.476 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.117453263Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-26T04:54:22.476 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.117875349Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-04-26T04:54:22.476 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.118285447Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-04-26T04:54:22.476 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.11867423Z level=info msg="Executing migration" id="Drop public config table" 2024-04-26T04:54:22.476 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.11908623Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-04-26T04:54:22.476 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.119489891Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-04-26T04:54:22.476 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.119891187Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-04-26T04:54:22.476 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.120331162Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-04-26T04:54:22.476 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.121195595Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-04-26T04:54:22.476 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.124120401Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-04-26T04:54:22.476 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.125109628Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-04-26T04:54:22.476 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.126097245Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-04-26T04:54:22.476 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.126406675Z level=info msg="Executing migration" id="add share column" 2024-04-26T04:54:22.477 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.12746869Z level=info msg="Executing migration" id="create default alerting folders" 2024-04-26T04:54:22.477 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.127896333Z level=info msg="Executing migration" id="create file table" 2024-04-26T04:54:22.477 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.128320901Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-04-26T04:54:22.477 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.129159562Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-04-26T04:54:22.477 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.129624701Z level=info msg="Executing migration" id="create file_meta table" 2024-04-26T04:54:22.477 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.130648957Z level=info msg="Executing migration" id="file table idx: path key" 2024-04-26T04:54:22.477 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.131327546Z level=info msg="Executing migration" id="set path collation in file table" 2024-04-26T04:54:22.477 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.132338498Z level=info msg="Executing migration" id="managed permissions migration" 2024-04-26T04:54:22.477 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.133044655Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-04-26T04:54:22.477 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.13406153Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-04-26T04:54:22.477 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.134867851Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-04-26T04:54:22.477 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.136064901Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-04-26T04:54:22.478 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.137030001Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-04-26T04:54:22.478 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.137777819Z level=info msg="Executing migration" id="update group index for alert rules" 2024-04-26T04:54:22.478 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.140649884Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-04-26T04:54:22.478 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.143458032Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-04-26T04:54:22.478 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.146183842Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-04-26T04:54:22.478 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.148783211Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-04-26T04:54:22.478 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.151683068Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-04-26T04:54:22.478 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.154297929Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-04-26T04:54:22.478 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.16404736Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-04-26T04:54:22.478 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.166725795Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-04-26T04:54:22.478 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.169684558Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-04-26T04:54:22.478 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.173130768Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-04-26T04:54:22.478 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.176001714Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-04-26T04:54:22.478 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.178654373Z level=info msg="Executing migration" id="create folder table" 2024-04-26T04:54:22.479 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.181275648Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-04-26T04:54:22.479 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.184123015Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-04-26T04:54:22.479 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.186992402Z level=info msg="Executing migration" id="Update folder title length" 2024-04-26T04:54:22.479 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.189811458Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-04-26T04:54:22.479 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=migrator t=2024-04-26T04:54:22.192590085Z level=info msg="migrations completed" performed=485 skipped=0 duration=724.994934ms 2024-04-26T04:54:22.479 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=sqlstore t=2024-04-26T04:54:22.193152348Z level=info msg="Created default organization" 2024-04-26T04:54:22.479 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=plugin.loader t=2024-04-26T04:54:22.251034783Z level=info msg="Plugin registered" pluginID=input 2024-04-26T04:54:22.479 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=plugin.loader t=2024-04-26T04:54:22.277475174Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-04-26T04:54:22.479 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=plugin.loader t=2024-04-26T04:54:22.277491813Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-04-26T04:54:22.479 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=secrets t=2024-04-26T04:54:22.277523683Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-04-26T04:54:22.479 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=query_data t=2024-04-26T04:54:22.279184086Z level=info msg="Query Service initialization" 2024-04-26T04:54:22.479 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=live.push_http t=2024-04-26T04:54:22.281663802Z level=info msg="Live Push Gateway initialization" 2024-04-26T04:54:22.723 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:22.723 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:22.723 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-mon[29897]: Deploying daemon node-exporter.a on smithi012 2024-04-26T04:54:22.723 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:22.723 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:22 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:22.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:22 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:22.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:22 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:22.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:22 smithi012 ceph-mon[37567]: Deploying daemon node-exporter.a on smithi012 2024-04-26T04:54:22.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:22 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:22.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:22 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:22.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:22 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:22.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:22 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:22.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:22 smithi012 ceph-mon[26617]: Deploying daemon node-exporter.a on smithi012 2024-04-26T04:54:22.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:22 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:22.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:22 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:23.492 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:23 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=infra.usagestats.collector t=2024-04-26T04:54:23.038296856Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-04-26T04:54:23.492 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:23 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=provisioning.datasources t=2024-04-26T04:54:23.038671128Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-04-26T04:54:23.492 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:23 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=provisioning.datasources t=2024-04-26T04:54:23.069961574Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-04-26T04:54:23.492 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:23 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=provisioning.datasources t=2024-04-26T04:54:23.11172952Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-04-26T04:54:23.492 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:23 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=provisioning.alerting t=2024-04-26T04:54:23.141259355Z level=info msg="starting to provision alerting" 2024-04-26T04:54:23.492 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:23 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=provisioning.alerting t=2024-04-26T04:54:23.141293887Z level=info msg="finished to provision alerting" 2024-04-26T04:54:23.492 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:23 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=ngalert.state.manager t=2024-04-26T04:54:23.141442591Z level=info msg="Warming state cache for startup" 2024-04-26T04:54:23.492 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:23 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=grafanaStorageLogger t=2024-04-26T04:54:23.141605138Z level=info msg="storage starting" 2024-04-26T04:54:23.492 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:23 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=http.server t=2024-04-26T04:54:23.144488848Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-04-26T04:54:23.492 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:23 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=sqlstore.transactions t=2024-04-26T04:54:23.152284933Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-26T04:54:23.493 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:23 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=ngalert.state.manager t=2024-04-26T04:54:23.152388632Z level=info msg="State cache has been initialized" states=0 duration=10.943686ms 2024-04-26T04:54:23.493 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:23 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=ticker t=2024-04-26T04:54:23.152469325Z level=info msg=starting first_tick=2024-04-26T04:54:30Z 2024-04-26T04:54:23.493 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:54:23 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[59109]: logger=ngalert.multiorg.alertmanager t=2024-04-26T04:54:23.152472376Z level=info msg="starting MultiOrg Alertmanager" 2024-04-26T04:54:23.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:23 smithi012 ceph-mon[26617]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:23.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:23 smithi012 ceph-mon[37567]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:23.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:23 smithi116 ceph-mon[29897]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:23.992 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:54:23 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:54:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:54:25.812 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:25 smithi012 ceph-mon[26617]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:25.812 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:25 smithi012 ceph-mon[37567]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:25.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:25 smithi116 ceph-mon[29897]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:26.878 INFO:teuthology.orchestra.run.smithi012.stdout:201863462931 2024-04-26T04:54:26.879 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-26T04:54:27.460 INFO:teuthology.orchestra.run.smithi012.stdout:133143986204 2024-04-26T04:54:27.460 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-26T04:54:27.703 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:27 smithi012 ceph-mon[26617]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:27.703 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:27 smithi012 ceph-mon[37567]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:27.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:27 smithi116 ceph-mon[29897]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:29.082 INFO:teuthology.orchestra.run.smithi012.stdout:55834574891 2024-04-26T04:54:29.083 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-26T04:54:29.700 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:29 smithi012 ceph-mon[26617]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:29.700 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:29 smithi012 ceph-mon[37567]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:29.793 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:29 smithi116 ceph-mon[29897]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:29.876 INFO:teuthology.orchestra.run.smithi012.stdout:77309411366 2024-04-26T04:54:29.876 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-26T04:54:30.563 INFO:teuthology.orchestra.run.smithi012.stdout:111669149727 2024-04-26T04:54:30.564 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-26T04:54:31.275 INFO:teuthology.orchestra.run.smithi012.stdout:236223201295 2024-04-26T04:54:31.275 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-26T04:54:31.863 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:31 smithi012 ceph-mon[37567]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:31.863 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:31 smithi012 ceph-mon[26617]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:31.908 INFO:teuthology.orchestra.run.smithi012.stdout:167503724569 2024-04-26T04:54:31.909 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-26T04:54:31.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:31 smithi116 ceph-mon[29897]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:32.282 INFO:teuthology.orchestra.run.smithi012.stdout:34359738415 2024-04-26T04:54:32.282 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-26T04:54:32.300 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:32.304 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:32.350 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:33.711 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:33 smithi012 ceph-mon[26617]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:33.712 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:33 smithi012 ceph-mon[37567]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:33.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:33 smithi116 ceph-mon[29897]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:33.991 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:54:33 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:54:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:54:35.934 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:35.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:35 smithi116 ceph-mon[29897]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:36.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:35 smithi012 ceph-mon[26617]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:36.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:35 smithi012 ceph-mon[37567]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:36.137 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:36.137 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:36.137 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:37.759 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:37 smithi012 ceph-mon[26617]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:37.759 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:37 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:37.759 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:37 smithi012 ceph-mon[37567]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:37.759 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:37 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:37.979 INFO:teuthology.orchestra.run.smithi012.stdout:201863462933 2024-04-26T04:54:37.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:37 smithi116 ceph-mon[29897]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:37.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:37 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:38.177 INFO:teuthology.orchestra.run.smithi012.stdout:167503724570 2024-04-26T04:54:38.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:38 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1630954916' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T04:54:38.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:38 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/2342078031' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T04:54:38.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:38 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/1630954916' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T04:54:38.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:38 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/2342078031' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T04:54:38.892 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724569 got 167503724570 for osd.5 2024-04-26T04:54:38.893 DEBUG:teuthology.parallel:result is None 2024-04-26T04:54:38.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:38 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/1630954916' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T04:54:38.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:38 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/2342078031' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T04:54:39.613 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:39 smithi012 ceph-mon[26617]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:39.613 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:39 smithi012 ceph-mon[37567]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:39.844 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:39 smithi116 ceph-mon[29897]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:40.357 INFO:teuthology.orchestra.run.smithi012.stdout:77309411369 2024-04-26T04:54:40.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:40 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/3302566646' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T04:54:41.007 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:40 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/3302566646' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T04:54:41.007 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:40 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/3302566646' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T04:54:41.652 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:41 smithi012 ceph-mon[26617]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:41.652 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:41 smithi012 ceph-mon[37567]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:41.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:41 smithi116 ceph-mon[29897]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:42.208 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411366 got 77309411369 for osd.2 2024-04-26T04:54:42.208 DEBUG:teuthology.parallel:result is None 2024-04-26T04:54:42.714 INFO:teuthology.orchestra.run.smithi012.stdout:236223201297 2024-04-26T04:54:43.296 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201295 got 236223201297 for osd.7 2024-04-26T04:54:43.296 DEBUG:teuthology.parallel:result is None 2024-04-26T04:54:43.918 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:43 smithi012 ceph-mon[26617]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:43.918 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:43 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/3679627569' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T04:54:43.919 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:43 smithi012 ceph-mon[37567]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:43.919 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:43 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/3679627569' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T04:54:43.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:43 smithi116 ceph-mon[29897]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:43.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:43 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/3679627569' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T04:54:43.992 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:54:43 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:54:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:54:44.238 INFO:teuthology.orchestra.run.smithi012.stdout:133143986207 2024-04-26T04:54:44.857 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:44 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1603836662' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T04:54:44.858 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:44 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/1603836662' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T04:54:44.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:44 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/1603836662' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T04:54:45.512 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:54:45 smithi012 systemd[1]: Started Ceph node-exporter.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:54:45.882 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:45 smithi012 ceph-mon[26617]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:45.882 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:45 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:45.883 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:45 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:45.883 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:45 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:45.883 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:45 smithi012 ceph-mon[37567]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:45.883 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:45 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:45.883 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:45 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:45.883 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:45 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:45.915 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462931 got 201863462933 for osd.6 2024-04-26T04:54:45.915 DEBUG:teuthology.parallel:result is None 2024-04-26T04:54:45.976 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:45 smithi116 ceph-mon[29897]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:45.976 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:45 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:45.976 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:45 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:45.976 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:45 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:46.241 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:46 smithi116 bash[59382]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-04-26T04:54:46.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:46 smithi012 ceph-mon[26617]: Deploying daemon node-exporter.b on smithi116 2024-04-26T04:54:46.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:46 smithi012 ceph-mon[37567]: Deploying daemon node-exporter.b on smithi116 2024-04-26T04:54:46.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:46 smithi116 ceph-mon[29897]: Deploying daemon node-exporter.b on smithi116 2024-04-26T04:54:46.992 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:46 smithi116 bash[59382]: Getting image source signatures 2024-04-26T04:54:46.992 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:46 smithi116 bash[59382]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-04-26T04:54:46.992 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:46 smithi116 bash[59382]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-04-26T04:54:46.992 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:46 smithi116 bash[59382]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-04-26T04:54:47.007 INFO:teuthology.orchestra.run.smithi012.stdout:55834574895 2024-04-26T04:54:47.212 INFO:teuthology.orchestra.run.smithi012.stdout:111669149732 2024-04-26T04:54:47.524 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:47.862 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:47 smithi012 ceph-mon[37567]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:47.863 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:47 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/4182008882' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T04:54:47.863 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:47 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/714295108' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T04:54:47.863 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:47 smithi012 ceph-mon[26617]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:47.863 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:47 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/4182008882' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T04:54:47.863 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:47 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/714295108' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T04:54:47.889 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986204 got 133143986207 for osd.4 2024-04-26T04:54:47.889 DEBUG:teuthology.parallel:result is None 2024-04-26T04:54:47.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:47 smithi116 ceph-mon[29897]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:47.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:47 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/4182008882' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T04:54:47.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:47 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/714295108' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T04:54:48.210 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149727 got 111669149732 for osd.3 2024-04-26T04:54:48.211 DEBUG:teuthology.parallel:result is None 2024-04-26T04:54:48.491 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 bash[59382]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-04-26T04:54:48.491 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 bash[59382]: Writing manifest to image destination 2024-04-26T04:54:48.672 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574891 got 55834574895 for osd.1 2024-04-26T04:54:48.673 DEBUG:teuthology.parallel:result is None 2024-04-26T04:54:48.766 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 podman[59382]: 2024-04-26 04:54:48.481091777 +0000 UTC m=+2.402733867 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-26T04:54:48.766 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 podman[59382]: 2024-04-26 04:54:48.588261418 +0000 UTC m=+2.509903504 container create da87ef4e90ea0c2a7856ff12c1f97cf903fd1151c791917236dce03e6e9cb1fa (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:54:48.766 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 podman[59382]: 2024-04-26 04:54:48.765720285 +0000 UTC m=+2.687362372 container init da87ef4e90ea0c2a7856ff12c1f97cf903fd1151c791917236dce03e6e9cb1fa (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:54:49.242 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 podman[59382]: 2024-04-26 04:54:48.769011883 +0000 UTC m=+2.690653972 container start da87ef4e90ea0c2a7856ff12c1f97cf903fd1151c791917236dce03e6e9cb1fa (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:54:49.242 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.774Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-26T04:54:49.242 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.774Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-26T04:54:49.242 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.774Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-26T04:54:49.242 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.774Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-26T04:54:49.242 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-26T04:54:49.243 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-26T04:54:49.243 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-26T04:54:49.243 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=arp 2024-04-26T04:54:49.243 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-26T04:54:49.243 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-26T04:54:49.243 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-26T04:54:49.243 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-26T04:54:49.243 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-26T04:54:49.243 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-26T04:54:49.243 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-26T04:54:49.243 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-26T04:54:49.243 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=edac 2024-04-26T04:54:49.244 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-26T04:54:49.244 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-26T04:54:49.244 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-26T04:54:49.244 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-26T04:54:49.244 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-26T04:54:49.244 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-26T04:54:49.244 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-26T04:54:49.244 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-26T04:54:49.244 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-26T04:54:49.244 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-26T04:54:49.244 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-26T04:54:49.244 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-26T04:54:49.244 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-26T04:54:49.244 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-26T04:54:49.245 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-26T04:54:49.245 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-26T04:54:49.245 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=os 2024-04-26T04:54:49.245 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-26T04:54:49.245 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-26T04:54:49.245 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-26T04:54:49.245 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-26T04:54:49.245 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-26T04:54:49.245 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-26T04:54:49.245 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-26T04:54:49.246 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=stat 2024-04-26T04:54:49.246 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-26T04:54:49.246 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-26T04:54:49.246 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.775Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-26T04:54:49.246 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.776Z caller=node_exporter.go:117 level=info collector=time 2024-04-26T04:54:49.246 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.776Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-26T04:54:49.246 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.776Z caller=node_exporter.go:117 level=info collector=uname 2024-04-26T04:54:49.246 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.776Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-26T04:54:49.246 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.776Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-26T04:54:49.246 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.776Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-26T04:54:49.246 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.776Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-26T04:54:49.246 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[59434]: ts=2024-04-26T04:54:48.776Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-26T04:54:49.247 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 bash[59382]: da87ef4e90ea0c2a7856ff12c1f97cf903fd1151c791917236dce03e6e9cb1fa 2024-04-26T04:54:49.247 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:54:48 smithi116 systemd[1]: Started Ceph node-exporter.b for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:54:49.898 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:49 smithi012 ceph-mon[26617]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:49.898 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:49 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:49.898 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:49 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:49.898 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:49 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:49.898 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:49 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:49.898 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:49 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:49.898 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:49 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:49.899 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:49 smithi012 ceph-mon[37567]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:49.899 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:49 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:49.899 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:49 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:49.899 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:49 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:49.899 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:49 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:49.899 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:49 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:49.899 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:49 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:49.910 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:49 smithi116 ceph-mon[29897]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:49.910 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:49 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:49.911 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:49 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:49.911 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:49 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:49.911 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:49 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:49.911 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:49 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:49.911 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:49 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:50.465 INFO:teuthology.orchestra.run.smithi012.stdout:34359738420 2024-04-26T04:54:50.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:50 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1695701699' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T04:54:50.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:50 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/1695701699' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T04:54:50.856 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:50 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/1695701699' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T04:54:51.305 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738415 got 34359738420 for osd.0 2024-04-26T04:54:51.305 DEBUG:teuthology.parallel:result is None 2024-04-26T04:54:51.306 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-26T04:54:51.306 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-26T04:54:51.762 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:51.886 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:51 smithi012 ceph-mon[26617]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:51.886 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:51 smithi012 ceph-mon[37567]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:51.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:51 smithi116 ceph-mon[29897]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:52.893 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:52 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:52.893 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:52 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:52.893 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:52 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:52.893 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:52 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:52.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:52 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:52.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:52 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:53.548 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:54:53.550 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2024-04-26T04:54:53.802 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:53 smithi012 ceph-mon[26617]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:53.802 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:53 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:53.802 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:53 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:53.802 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:53 smithi012 ceph-mon[37567]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:53.802 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:53 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:53.802 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:53 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:53.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:53 smithi116 ceph-mon[29897]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:53.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:53 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:53.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:53 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:53.992 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:54:53 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:54:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:54:54.751 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:54 smithi012 ceph-mon[26617]: from='client.24527 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:54:54.751 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:54 smithi012 ceph-mon[37567]: from='client.24527 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:54:54.778 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":37,"stamp":"2024-04-26T04:54:52.415657+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385572,"kb_used_data":6712,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575644,"statfs":{"total":767960285184,"available":765517459456,"internally_reserved":0,"allocated":6873088,"data_stored":3897991,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001880"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:54:00.103372+0000","last_change":"2024-04-26T04:53:57.857083+0000","last_active":"2024-04-26T04:54:00.103372+0000","last_peered":"2024-04-26T04:54:00.103372+0000","last_clean":"2024-04-26T04:54:00.103372+0000","last_became_active":"2024-04-26T04:53:57.856775+0000","last_became_peered":"2024-04-26T04:53:57.856775+0000","last_unstale":"2024-04-26T04:54:00.103372+0000","last_undegraded":"2024-04-26T04:54:00.103372+0000","last_fullsized":"2024-04-26T04:54:00.103372+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:06:53.087376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:53:58.857858+0000","last_change":"2024-04-26T04:53:57.938166+0000","last_active":"2024-04-26T04:53:58.857858+0000","last_peered":"2024-04-26T04:53:58.857858+0000","last_clean":"2024-04-26T04:53:58.857858+0000","last_became_active":"2024-04-26T04:53:57.937842+0000","last_became_peered":"2024-04-26T04:53:57.937842+0000","last_unstale":"2024-04-26T04:53:58.857858+0000","last_undegraded":"2024-04-26T04:53:58.857858+0000","last_fullsized":"2024-04-26T04:53:58.857858+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:39:45.320158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:53:58.860510+0000","last_change":"2024-04-26T04:53:57.939093+0000","last_active":"2024-04-26T04:53:58.860510+0000","last_peered":"2024-04-26T04:53:58.860510+0000","last_clean":"2024-04-26T04:53:58.860510+0000","last_became_active":"2024-04-26T04:53:57.938662+0000","last_became_peered":"2024-04-26T04:53:57.938662+0000","last_unstale":"2024-04-26T04:53:58.860510+0000","last_undegraded":"2024-04-26T04:53:58.860510+0000","last_fullsized":"2024-04-26T04:53:58.860510+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:37:22.972048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:53:58.860494+0000","last_change":"2024-04-26T04:53:57.939087+0000","last_active":"2024-04-26T04:53:58.860494+0000","last_peered":"2024-04-26T04:53:58.860494+0000","last_clean":"2024-04-26T04:53:58.860494+0000","last_became_active":"2024-04-26T04:53:57.938647+0000","last_became_peered":"2024-04-26T04:53:57.938647+0000","last_unstale":"2024-04-26T04:53:58.860494+0000","last_undegraded":"2024-04-26T04:53:58.860494+0000","last_fullsized":"2024-04-26T04:53:58.860494+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:51:38.687896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":271,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:53:58.860583+0000","last_change":"2024-04-26T04:53:25.550145+0000","last_active":"2024-04-26T04:53:58.860583+0000","last_peered":"2024-04-26T04:53:58.860583+0000","last_clean":"2024-04-26T04:53:58.860583+0000","last_became_active":"2024-04-26T04:53:25.549600+0000","last_became_peered":"2024-04-26T04:53:25.549600+0000","last_unstale":"2024-04-26T04:53:58.860583+0000","last_undegraded":"2024-04-26T04:53:58.860583+0000","last_fullsized":"2024-04-26T04:53:58.860583+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:51:27.309540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:51:27.309540+0000","last_clean_scrub_stamp":"2024-04-26T04:51:27.309540+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:23:44.865329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:54:00.103346+0000","last_change":"2024-04-26T04:53:57.856241+0000","last_active":"2024-04-26T04:54:00.103346+0000","last_peered":"2024-04-26T04:54:00.103346+0000","last_clean":"2024-04-26T04:54:00.103346+0000","last_became_active":"2024-04-26T04:53:57.855745+0000","last_became_peered":"2024-04-26T04:53:57.855745+0000","last_unstale":"2024-04-26T04:54:00.103346+0000","last_undegraded":"2024-04-26T04:54:00.103346+0000","last_fullsized":"2024-04-26T04:54:00.103346+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:54:03.208950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:54:00.102986+0000","last_change":"2024-04-26T04:53:57.939187+0000","last_active":"2024-04-26T04:54:00.102986+0000","last_peered":"2024-04-26T04:54:00.102986+0000","last_clean":"2024-04-26T04:54:00.102986+0000","last_became_active":"2024-04-26T04:53:57.938749+0000","last_became_peered":"2024-04-26T04:53:57.938749+0000","last_unstale":"2024-04-26T04:54:00.102986+0000","last_undegraded":"2024-04-26T04:54:00.102986+0000","last_fullsized":"2024-04-26T04:54:00.102986+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:59:54.974432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:54:00.103346+0000","last_change":"2024-04-26T04:53:57.856053+0000","last_active":"2024-04-26T04:54:00.103346+0000","last_peered":"2024-04-26T04:54:00.103346+0000","last_clean":"2024-04-26T04:54:00.103346+0000","last_became_active":"2024-04-26T04:53:57.855693+0000","last_became_peered":"2024-04-26T04:53:57.855693+0000","last_unstale":"2024-04-26T04:54:00.103346+0000","last_undegraded":"2024-04-26T04:54:00.103346+0000","last_fullsized":"2024-04-26T04:54:00.103346+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:35:20.404397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:53:58.857755+0000","last_change":"2024-04-26T04:53:57.939812+0000","last_active":"2024-04-26T04:53:58.857755+0000","last_peered":"2024-04-26T04:53:58.857755+0000","last_clean":"2024-04-26T04:53:58.857755+0000","last_became_active":"2024-04-26T04:53:57.939509+0000","last_became_peered":"2024-04-26T04:53:57.939509+0000","last_unstale":"2024-04-26T04:53:58.857755+0000","last_undegraded":"2024-04-26T04:53:58.857755+0000","last_fullsized":"2024-04-26T04:53:58.857755+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:05:41.872666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201299,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298556,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446596,"statfs":{"total":95995035648,"available":95689314304,"internally_reserved":0,"allocated":1228800,"data_stored":856087,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:54:23 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.629},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.55400000000000005}]},{"osd":1,"last update":"Fri Apr 26 04:54:23 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.64700000000000002}]},{"osd":2,"last update":"Fri Apr 26 04:54:23 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.56200000000000006}]},{"osd":3,"last update":"Fri Apr 26 04:54:23 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.67100000000000004}]},{"osd":4,"last update":"Fri Apr 26 04:54:23 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.371}]},{"osd":5,"last update":"Fri Apr 26 04:54:23 2024","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.38800000000000001}]},{"osd":6,"last update":"Fri Apr 26 04:54:23 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.432}]}]},{"osd":6,"up_from":47,"seq":201863462936,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298552,"kb_used_data":1196,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446600,"statfs":{"total":95995035648,"available":95689318400,"internally_reserved":0,"allocated":1224704,"data_stored":855872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:53:57 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.70599999999999996}]},{"osd":1,"last update":"Fri Apr 26 04:53:57 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.64100000000000001}]},{"osd":2,"last update":"Fri Apr 26 04:53:57 2024","interfaces":[{"interface":"back","average":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":5.6980000000000004,"5min":5.6980000000000004,"15min":5.6980000000000004},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.65800000000000003}]},{"osd":3,"last update":"Fri Apr 26 04:53:57 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":5.6369999999999996,"5min":5.6369999999999996,"15min":5.6369999999999996},"last":0.60299999999999998}]},{"osd":4,"last update":"Fri Apr 26 04:53:57 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"last":0.46500000000000002}]},{"osd":5,"last update":"Fri Apr 26 04:53:57 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.443},{"interface":"front","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.51700000000000002}]},{"osd":7,"last update":"Fri Apr 26 04:54:23 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.48799999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574896,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:54:08 2024","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.34399999999999997,"15min":0.32100000000000001},"min":{"1min":0.26300000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.35699999999999998,"15min":0.32900000000000001},"min":{"1min":0.317,"5min":0.214,"15min":0.214},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.46400000000000002}]},{"osd":2,"last update":"Fri Apr 26 04:54:27 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.39300000000000002,"15min":0.38600000000000001},"min":{"1min":0.28699999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.42199999999999999,"15min":0.41599999999999998},"min":{"1min":0.29899999999999999,"5min":0.22,"15min":0.22},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.38600000000000001}]},{"osd":3,"last update":"Fri Apr 26 04:53:53 2024","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.41499999999999998,"15min":0.41299999999999998},"min":{"1min":0.29399999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.38900000000000001,"5min":0.315,"15min":0.315},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.53400000000000003}]},{"osd":4,"last update":"Fri Apr 26 04:54:15 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.55400000000000005,"15min":0.54400000000000004},"min":{"1min":0.42599999999999999,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.99299999999999999,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.54200000000000004,"15min":0.53200000000000003},"min":{"1min":0.40799999999999997,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.51400000000000001}]},{"osd":5,"last update":"Fri Apr 26 04:54:41 2024","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.60399999999999998,"15min":0.59899999999999998},"min":{"1min":0.432,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.61499999999999999,"15min":0.61099999999999999},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.61199999999999999}]},{"osd":6,"last update":"Fri Apr 26 04:54:00 2024","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.0269999999999999,"5min":1.0269999999999999,"15min":1.0269999999999999},"last":0.66200000000000003},{"interface":"front","average":{"1min":1.907,"5min":1.907,"15min":1.907},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":24.962,"5min":24.962,"15min":24.962},"last":0.624}]},{"osd":7,"last update":"Fri Apr 26 04:54:24 2024","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"min":{"1min":0.495,"5min":0.495,"15min":0.495},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.63300000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738420,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298556,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446596,"statfs":{"total":95995035648,"available":95689314304,"internally_reserved":0,"allocated":1228800,"data_stored":856145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 26 04:54:09 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.38600000000000001,"15min":0.36599999999999999},"min":{"1min":0.26300000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.56999999999999995,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.25800000000000001},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.39800000000000002,"15min":0.38400000000000001},"min":{"1min":0.27900000000000003,"5min":0.22,"15min":0.22},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.29799999999999999}]},{"osd":2,"last update":"Fri Apr 26 04:54:34 2024","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.41199999999999998,"15min":0.40600000000000003},"min":{"1min":0.26900000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.59799999999999998,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.37},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.29499999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.65600000000000003,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.32300000000000001}]},{"osd":3,"last update":"Fri Apr 26 04:53:53 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.48799999999999999,"15min":0.49099999999999999},"min":{"1min":0.34499999999999997,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.63600000000000001,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.497,"5min":0.51000000000000001,"15min":0.51200000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.39500000000000002}]},{"osd":4,"last update":"Fri Apr 26 04:54:17 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.63700000000000001,"15min":0.64200000000000002},"min":{"1min":0.46300000000000002,"5min":0.44,"15min":0.44},"max":{"1min":0.86399999999999999,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.443},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.60399999999999998,"15min":0.60699999999999998},"min":{"1min":0.42499999999999999,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.79300000000000004,"5min":0.872,"15min":0.872},"last":0.497}]},{"osd":5,"last update":"Fri Apr 26 04:54:36 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.62,"15min":0.624},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.85599999999999998,"5min":0.88,"15min":0.88},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.64000000000000001,"15min":0.64600000000000002},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.77000000000000002,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.47699999999999998}]},{"osd":6,"last update":"Fri Apr 26 04:53:58 2024","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.504},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.41799999999999998}]},{"osd":7,"last update":"Fri Apr 26 04:54:25 2024","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.51800000000000002}]}]},{"osd":2,"up_from":18,"seq":77309411371,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297984,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447168,"statfs":{"total":95995035648,"available":95689900032,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:54:31 2024","interfaces":[{"interface":"back","average":{"1min":0.34000000000000002,"5min":0.38200000000000001,"15min":0.38700000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.27000000000000002},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.45300000000000001,"15min":0.44},"min":{"1min":0.33400000000000002,"5min":0.307,"15min":0.307},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.33600000000000002}]},{"osd":1,"last update":"Fri Apr 26 04:54:31 2024","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.41299999999999998,"15min":0.41399999999999998},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.52800000000000002,"5min":0.626,"15min":0.626},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.43099999999999999,"15min":0.42499999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.626,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.36199999999999999}]},{"osd":3,"last update":"Fri Apr 26 04:53:58 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.49399999999999999,"15min":0.48799999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.98999999999999999,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.52100000000000002,"15min":0.52300000000000002},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.38400000000000001}]},{"osd":4,"last update":"Fri Apr 26 04:54:14 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.59299999999999997,"15min":0.59799999999999998},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.79200000000000004,"5min":1.042,"15min":1.042},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.623,"15min":0.63},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.92200000000000004,"5min":1.1240000000000001,"15min":1.1240000000000001},"last":0.42799999999999999}]},{"osd":5,"last update":"Fri Apr 26 04:54:40 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.65900000000000003,"15min":0.67100000000000004},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.75800000000000001,"5min":1.077,"15min":1.077},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.622,"15min":0.629},"min":{"1min":0.439,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.78300000000000003,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.42099999999999999}]},{"osd":6,"last update":"Fri Apr 26 04:53:58 2024","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":1.012,"5min":1.012,"15min":1.012},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"min":{"1min":0.498,"5min":0.498,"15min":0.498},"max":{"1min":1.0960000000000001,"5min":1.0960000000000001,"15min":1.0960000000000001},"last":0.47499999999999998}]},{"osd":7,"last update":"Fri Apr 26 04:54:21 2024","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"min":{"1min":0.505,"5min":0.505,"15min":0.505},"max":{"1min":0.998,"5min":0.998,"15min":0.998},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.45400000000000001}]}]},{"osd":3,"up_from":26,"seq":111669149733,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:53:56 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.46300000000000002,"15min":0.46700000000000003},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.67300000000000004,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.437,"5min":0.45700000000000002,"15min":0.46100000000000002},"min":{"1min":0.33500000000000002,"5min":0.307,"15min":0.307},"max":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.373}]},{"osd":1,"last update":"Fri Apr 26 04:53:56 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.48699999999999999,"15min":0.495},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.58599999999999997,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.48199999999999998,"15min":0.48799999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.44600000000000001}]},{"osd":2,"last update":"Fri Apr 26 04:53:56 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.50600000000000001,"15min":0.50700000000000001},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.64600000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.436,"5min":0.504,"15min":0.51500000000000001},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.39300000000000002}]},{"osd":4,"last update":"Fri Apr 26 04:54:16 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.61499999999999999,"15min":0.61899999999999999},"min":{"1min":0.437,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.81000000000000005,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.74299999999999999},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.59499999999999997,"15min":0.60199999999999998},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.73199999999999998,"5min":0.745,"15min":0.745},"last":0.81200000000000006}]},{"osd":5,"last update":"Fri Apr 26 04:54:39 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":1.3009999999999999,"15min":1.4330000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.68100000000000005,"5min":18.239000000000001,"15min":18.239000000000001},"last":0.75600000000000001},{"interface":"front","average":{"1min":0.622,"5min":0.65400000000000003,"15min":0.66000000000000003},"min":{"1min":0.48199999999999998,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.85899999999999999,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.82899999999999996}]},{"osd":6,"last update":"Fri Apr 26 04:53:58 2024","interfaces":[{"interface":"back","average":{"1min":1.6379999999999999,"5min":1.6379999999999999,"15min":1.6379999999999999},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":18.221,"5min":18.221,"15min":18.221},"last":0.79200000000000004},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.77100000000000002}]},{"osd":7,"last update":"Fri Apr 26 04:54:26 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.72499999999999998},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"min":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.85099999999999998}]}]},{"osd":4,"up_from":31,"seq":133143986209,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297984,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447168,"statfs":{"total":95995035648,"available":95689900032,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:54:15 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.499,"15min":0.49299999999999999},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.58499999999999996}]},{"osd":1,"last update":"Fri Apr 26 04:54:15 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.50600000000000001,"15min":0.5},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.56100000000000005}]},{"osd":2,"last update":"Fri Apr 26 04:54:15 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.52900000000000003,"15min":0.53100000000000003},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.70599999999999996,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.50600000000000001,"15min":0.501},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.51900000000000002}]},{"osd":3,"last update":"Fri Apr 26 04:54:15 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.54500000000000004,"15min":0.54000000000000004},"min":{"1min":0.40000000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.54400000000000004,"15min":0.53800000000000003},"min":{"1min":0.38700000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.56999999999999995}]},{"osd":5,"last update":"Fri Apr 26 04:54:42 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.39800000000000002,"15min":0.38600000000000001},"min":{"1min":0.27200000000000002,"5min":0.221,"15min":0.221},"max":{"1min":0.72199999999999998,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.40699999999999997,"15min":0.39300000000000002},"min":{"1min":0.35799999999999998,"5min":0.251,"15min":0.251},"max":{"1min":0.66700000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.47099999999999997}]},{"osd":6,"last update":"Fri Apr 26 04:54:02 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.49199999999999999}]},{"osd":7,"last update":"Fri Apr 26 04:54:21 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.50700000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724573,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":634880,"data_stored":265719,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:54:37 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.54700000000000004,"15min":0.55200000000000005},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.70699999999999996,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.438},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.55500000000000005,"15min":0.55900000000000005},"min":{"1min":0.36699999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.71499999999999997,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.41399999999999998}]},{"osd":1,"last update":"Fri Apr 26 04:54:37 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.54700000000000004,"15min":0.55200000000000005},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.55900000000000005,"15min":0.56499999999999995},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.71499999999999997,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.38}]},{"osd":2,"last update":"Fri Apr 26 04:54:37 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.55800000000000005,"15min":0.56399999999999995},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.77000000000000002,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.58499999999999996,"15min":0.59399999999999997},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.36699999999999999}]},{"osd":3,"last update":"Fri Apr 26 04:54:37 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.61599999999999999,"15min":0.624},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.77400000000000002,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.60499999999999998,"15min":0.60899999999999999},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.46999999999999997}]},{"osd":4,"last update":"Fri Apr 26 04:54:37 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.40000000000000002,"15min":0.38800000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.41499999999999998,"15min":0.39900000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.307}]},{"osd":6,"last update":"Fri Apr 26 04:53:58 2024","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.39000000000000001}]},{"osd":7,"last update":"Fri Apr 26 04:54:24 2024","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.35399999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-26T04:54:54.780 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-26T04:54:54.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:54 smithi116 ceph-mon[29897]: from='client.24527 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:54:55.610 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:55.847 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:55 smithi012 ceph-mon[26617]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:55.847 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:55 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:55.847 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:55 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:55.847 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:55 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:55.847 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:55 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:54:55.848 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:55 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:55.848 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:55 smithi012 ceph-mon[37567]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:55.848 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:55 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:55.848 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:55 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:55.848 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:55 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:55.848 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:55 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:54:55.848 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:55 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:55.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:55 smithi116 ceph-mon[29897]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:55.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:55 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:55.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:55 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:55.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:55 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:55.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:55 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:54:55.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:55 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:54:57.662 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:57 smithi012 ceph-mon[26617]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-26T04:54:57.663 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:57 smithi012 ceph-mon[26617]: Reconfiguring daemon alertmanager.a on smithi012 2024-04-26T04:54:57.663 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:57 smithi012 ceph-mon[26617]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:57.663 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:57 smithi012 ceph-mon[37567]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-26T04:54:57.663 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:57 smithi012 ceph-mon[37567]: Reconfiguring daemon alertmanager.a on smithi012 2024-04-26T04:54:57.663 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:57 smithi012 ceph-mon[37567]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:57.899 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:54:57.899 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2024-04-26T04:54:57.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:57 smithi116 ceph-mon[29897]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-26T04:54:57.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:57 smithi116 ceph-mon[29897]: Reconfiguring daemon alertmanager.a on smithi012 2024-04-26T04:54:57.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:57 smithi116 ceph-mon[29897]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:58.519 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":39,"stamp":"2024-04-26T04:54:56.416218+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385572,"kb_used_data":6712,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575644,"statfs":{"total":767960285184,"available":765517459456,"internally_reserved":0,"allocated":6873088,"data_stored":3897991,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001864"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:54:00.103372+0000","last_change":"2024-04-26T04:53:57.857083+0000","last_active":"2024-04-26T04:54:00.103372+0000","last_peered":"2024-04-26T04:54:00.103372+0000","last_clean":"2024-04-26T04:54:00.103372+0000","last_became_active":"2024-04-26T04:53:57.856775+0000","last_became_peered":"2024-04-26T04:53:57.856775+0000","last_unstale":"2024-04-26T04:54:00.103372+0000","last_undegraded":"2024-04-26T04:54:00.103372+0000","last_fullsized":"2024-04-26T04:54:00.103372+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:06:53.087376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:53:58.857858+0000","last_change":"2024-04-26T04:53:57.938166+0000","last_active":"2024-04-26T04:53:58.857858+0000","last_peered":"2024-04-26T04:53:58.857858+0000","last_clean":"2024-04-26T04:53:58.857858+0000","last_became_active":"2024-04-26T04:53:57.937842+0000","last_became_peered":"2024-04-26T04:53:57.937842+0000","last_unstale":"2024-04-26T04:53:58.857858+0000","last_undegraded":"2024-04-26T04:53:58.857858+0000","last_fullsized":"2024-04-26T04:53:58.857858+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:39:45.320158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:53:58.860510+0000","last_change":"2024-04-26T04:53:57.939093+0000","last_active":"2024-04-26T04:53:58.860510+0000","last_peered":"2024-04-26T04:53:58.860510+0000","last_clean":"2024-04-26T04:53:58.860510+0000","last_became_active":"2024-04-26T04:53:57.938662+0000","last_became_peered":"2024-04-26T04:53:57.938662+0000","last_unstale":"2024-04-26T04:53:58.860510+0000","last_undegraded":"2024-04-26T04:53:58.860510+0000","last_fullsized":"2024-04-26T04:53:58.860510+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:37:22.972048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:53:58.860494+0000","last_change":"2024-04-26T04:53:57.939087+0000","last_active":"2024-04-26T04:53:58.860494+0000","last_peered":"2024-04-26T04:53:58.860494+0000","last_clean":"2024-04-26T04:53:58.860494+0000","last_became_active":"2024-04-26T04:53:57.938647+0000","last_became_peered":"2024-04-26T04:53:57.938647+0000","last_unstale":"2024-04-26T04:53:58.860494+0000","last_undegraded":"2024-04-26T04:53:58.860494+0000","last_fullsized":"2024-04-26T04:53:58.860494+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:51:38.687896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":271,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:53:58.860583+0000","last_change":"2024-04-26T04:53:25.550145+0000","last_active":"2024-04-26T04:53:58.860583+0000","last_peered":"2024-04-26T04:53:58.860583+0000","last_clean":"2024-04-26T04:53:58.860583+0000","last_became_active":"2024-04-26T04:53:25.549600+0000","last_became_peered":"2024-04-26T04:53:25.549600+0000","last_unstale":"2024-04-26T04:53:58.860583+0000","last_undegraded":"2024-04-26T04:53:58.860583+0000","last_fullsized":"2024-04-26T04:53:58.860583+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:51:27.309540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:51:27.309540+0000","last_clean_scrub_stamp":"2024-04-26T04:51:27.309540+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:23:44.865329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:54:00.103346+0000","last_change":"2024-04-26T04:53:57.856241+0000","last_active":"2024-04-26T04:54:00.103346+0000","last_peered":"2024-04-26T04:54:00.103346+0000","last_clean":"2024-04-26T04:54:00.103346+0000","last_became_active":"2024-04-26T04:53:57.855745+0000","last_became_peered":"2024-04-26T04:53:57.855745+0000","last_unstale":"2024-04-26T04:54:00.103346+0000","last_undegraded":"2024-04-26T04:54:00.103346+0000","last_fullsized":"2024-04-26T04:54:00.103346+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:54:03.208950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:54:00.102986+0000","last_change":"2024-04-26T04:53:57.939187+0000","last_active":"2024-04-26T04:54:00.102986+0000","last_peered":"2024-04-26T04:54:00.102986+0000","last_clean":"2024-04-26T04:54:00.102986+0000","last_became_active":"2024-04-26T04:53:57.938749+0000","last_became_peered":"2024-04-26T04:53:57.938749+0000","last_unstale":"2024-04-26T04:54:00.102986+0000","last_undegraded":"2024-04-26T04:54:00.102986+0000","last_fullsized":"2024-04-26T04:54:00.102986+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:59:54.974432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:54:00.103346+0000","last_change":"2024-04-26T04:53:57.856053+0000","last_active":"2024-04-26T04:54:00.103346+0000","last_peered":"2024-04-26T04:54:00.103346+0000","last_clean":"2024-04-26T04:54:00.103346+0000","last_became_active":"2024-04-26T04:53:57.855693+0000","last_became_peered":"2024-04-26T04:53:57.855693+0000","last_unstale":"2024-04-26T04:54:00.103346+0000","last_undegraded":"2024-04-26T04:54:00.103346+0000","last_fullsized":"2024-04-26T04:54:00.103346+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:35:20.404397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:53:58.857755+0000","last_change":"2024-04-26T04:53:57.939812+0000","last_active":"2024-04-26T04:53:58.857755+0000","last_peered":"2024-04-26T04:53:58.857755+0000","last_clean":"2024-04-26T04:53:58.857755+0000","last_became_active":"2024-04-26T04:53:57.939509+0000","last_became_peered":"2024-04-26T04:53:57.939509+0000","last_unstale":"2024-04-26T04:53:58.857755+0000","last_undegraded":"2024-04-26T04:53:58.857755+0000","last_fullsized":"2024-04-26T04:53:58.857755+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:05:41.872666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201300,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298556,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446596,"statfs":{"total":95995035648,"available":95689314304,"internally_reserved":0,"allocated":1228800,"data_stored":856087,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:54:23 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.379}]},{"osd":1,"last update":"Fri Apr 26 04:54:23 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.5},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.40899999999999997}]},{"osd":2,"last update":"Fri Apr 26 04:54:23 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.39600000000000002}]},{"osd":3,"last update":"Fri Apr 26 04:54:23 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.47899999999999998}]},{"osd":4,"last update":"Fri Apr 26 04:54:23 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.42599999999999999}]},{"osd":5,"last update":"Fri Apr 26 04:54:23 2024","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.438}]},{"osd":6,"last update":"Fri Apr 26 04:54:23 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.48899999999999999}]}]},{"osd":6,"up_from":47,"seq":201863462937,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298552,"kb_used_data":1196,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446600,"statfs":{"total":95995035648,"available":95689318400,"internally_reserved":0,"allocated":1224704,"data_stored":855872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:53:57 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.57499999999999996}]},{"osd":1,"last update":"Fri Apr 26 04:53:57 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.60699999999999998}]},{"osd":2,"last update":"Fri Apr 26 04:53:57 2024","interfaces":[{"interface":"back","average":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":5.6980000000000004,"5min":5.6980000000000004,"15min":5.6980000000000004},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.61899999999999999}]},{"osd":3,"last update":"Fri Apr 26 04:53:57 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":5.6369999999999996,"5min":5.6369999999999996,"15min":5.6369999999999996},"last":0.58699999999999997}]},{"osd":4,"last update":"Fri Apr 26 04:53:57 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"last":0.39100000000000001}]},{"osd":5,"last update":"Fri Apr 26 04:53:57 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.55100000000000005}]},{"osd":7,"last update":"Fri Apr 26 04:54:23 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.41799999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574897,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:54:08 2024","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.34399999999999997,"15min":0.32100000000000001},"min":{"1min":0.26300000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.35699999999999998,"15min":0.32900000000000001},"min":{"1min":0.317,"5min":0.214,"15min":0.214},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.28999999999999998}]},{"osd":2,"last update":"Fri Apr 26 04:54:27 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.39300000000000002,"15min":0.38600000000000001},"min":{"1min":0.28699999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.42199999999999999,"15min":0.41599999999999998},"min":{"1min":0.29899999999999999,"5min":0.22,"15min":0.22},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.32500000000000001}]},{"osd":3,"last update":"Fri Apr 26 04:53:53 2024","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.41499999999999998,"15min":0.41299999999999998},"min":{"1min":0.29399999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.38900000000000001,"5min":0.315,"15min":0.315},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.30599999999999999}]},{"osd":4,"last update":"Fri Apr 26 04:54:15 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.55400000000000005,"15min":0.54400000000000004},"min":{"1min":0.42599999999999999,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.99299999999999999,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.54200000000000004,"15min":0.53200000000000003},"min":{"1min":0.40799999999999997,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.46800000000000003}]},{"osd":5,"last update":"Fri Apr 26 04:54:41 2024","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.60399999999999998,"15min":0.59899999999999998},"min":{"1min":0.432,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.61499999999999999,"15min":0.61099999999999999},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.45100000000000001}]},{"osd":6,"last update":"Fri Apr 26 04:54:00 2024","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.0269999999999999,"5min":1.0269999999999999,"15min":1.0269999999999999},"last":0.47699999999999998},{"interface":"front","average":{"1min":1.907,"5min":1.907,"15min":1.907},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":24.962,"5min":24.962,"15min":24.962},"last":0.42999999999999999}]},{"osd":7,"last update":"Fri Apr 26 04:54:24 2024","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"min":{"1min":0.495,"5min":0.495,"15min":0.495},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.498}]}]},{"osd":0,"up_from":8,"seq":34359738421,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298556,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446596,"statfs":{"total":95995035648,"available":95689314304,"internally_reserved":0,"allocated":1228800,"data_stored":856145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 26 04:54:09 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.38600000000000001,"15min":0.36599999999999999},"min":{"1min":0.26300000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.56999999999999995,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.39800000000000002,"15min":0.38400000000000001},"min":{"1min":0.27900000000000003,"5min":0.22,"15min":0.22},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.33500000000000002}]},{"osd":2,"last update":"Fri Apr 26 04:54:34 2024","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.41199999999999998,"15min":0.40600000000000003},"min":{"1min":0.26900000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.59799999999999998,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.311},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.29499999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.65600000000000003,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.39000000000000001}]},{"osd":3,"last update":"Fri Apr 26 04:53:53 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.48799999999999999,"15min":0.49099999999999999},"min":{"1min":0.34499999999999997,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.63600000000000001,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.497,"5min":0.51000000000000001,"15min":0.51200000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.35899999999999999}]},{"osd":4,"last update":"Fri Apr 26 04:54:17 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.63700000000000001,"15min":0.64200000000000002},"min":{"1min":0.46300000000000002,"5min":0.44,"15min":0.44},"max":{"1min":0.86399999999999999,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.60399999999999998,"15min":0.60699999999999998},"min":{"1min":0.42499999999999999,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.79300000000000004,"5min":0.872,"15min":0.872},"last":0.44800000000000001}]},{"osd":5,"last update":"Fri Apr 26 04:54:36 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.62,"15min":0.624},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.85599999999999998,"5min":0.88,"15min":0.88},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.64000000000000001,"15min":0.64600000000000002},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.77000000000000002,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.53100000000000003}]},{"osd":6,"last update":"Fri Apr 26 04:53:58 2024","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.47799999999999998}]},{"osd":7,"last update":"Fri Apr 26 04:54:25 2024","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.47299999999999998}]}]},{"osd":2,"up_from":18,"seq":77309411372,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297984,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447168,"statfs":{"total":95995035648,"available":95689900032,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:54:31 2024","interfaces":[{"interface":"back","average":{"1min":0.34000000000000002,"5min":0.38200000000000001,"15min":0.38700000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.45300000000000001,"15min":0.44},"min":{"1min":0.33400000000000002,"5min":0.307,"15min":0.307},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.496}]},{"osd":1,"last update":"Fri Apr 26 04:54:31 2024","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.41299999999999998,"15min":0.41399999999999998},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.52800000000000002,"5min":0.626,"15min":0.626},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.43099999999999999,"15min":0.42499999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.626,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.45300000000000001}]},{"osd":3,"last update":"Fri Apr 26 04:53:58 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.49399999999999999,"15min":0.48799999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.98999999999999999,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.52100000000000002,"15min":0.52300000000000002},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.57599999999999996}]},{"osd":4,"last update":"Fri Apr 26 04:54:14 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.59299999999999997,"15min":0.59799999999999998},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.79200000000000004,"5min":1.042,"15min":1.042},"last":0.79900000000000004},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.623,"15min":0.63},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.92200000000000004,"5min":1.1240000000000001,"15min":1.1240000000000001},"last":0.67300000000000004}]},{"osd":5,"last update":"Fri Apr 26 04:54:40 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.65900000000000003,"15min":0.67100000000000004},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.75800000000000001,"5min":1.077,"15min":1.077},"last":0.74099999999999999},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.622,"15min":0.629},"min":{"1min":0.439,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.78300000000000003,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.68899999999999995}]},{"osd":6,"last update":"Fri Apr 26 04:53:58 2024","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":1.012,"5min":1.012,"15min":1.012},"last":0.77400000000000002},{"interface":"front","average":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"min":{"1min":0.498,"5min":0.498,"15min":0.498},"max":{"1min":1.0960000000000001,"5min":1.0960000000000001,"15min":1.0960000000000001},"last":0.76100000000000001}]},{"osd":7,"last update":"Fri Apr 26 04:54:21 2024","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"min":{"1min":0.505,"5min":0.505,"15min":0.505},"max":{"1min":0.998,"5min":0.998,"15min":0.998},"last":0.81200000000000006},{"interface":"front","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.72599999999999998}]}]},{"osd":3,"up_from":26,"seq":111669149734,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:53:56 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.46300000000000002,"15min":0.46700000000000003},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.67300000000000004,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.373},{"interface":"front","average":{"1min":0.437,"5min":0.45700000000000002,"15min":0.46100000000000002},"min":{"1min":0.33500000000000002,"5min":0.307,"15min":0.307},"max":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.36499999999999999}]},{"osd":1,"last update":"Fri Apr 26 04:53:56 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.48699999999999999,"15min":0.495},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.58599999999999997,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.48199999999999998,"15min":0.48799999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.32400000000000001}]},{"osd":2,"last update":"Fri Apr 26 04:53:56 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.50600000000000001,"15min":0.50700000000000001},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.64600000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.436,"5min":0.504,"15min":0.51500000000000001},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.44500000000000001}]},{"osd":4,"last update":"Fri Apr 26 04:54:16 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.61499999999999999,"15min":0.61899999999999999},"min":{"1min":0.437,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.81000000000000005,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.623},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.59499999999999997,"15min":0.60199999999999998},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.73199999999999998,"5min":0.745,"15min":0.745},"last":0.60899999999999999}]},{"osd":5,"last update":"Fri Apr 26 04:54:39 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":1.3009999999999999,"15min":1.4330000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.68100000000000005,"5min":18.239000000000001,"15min":18.239000000000001},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.622,"5min":0.65400000000000003,"15min":0.66000000000000003},"min":{"1min":0.48199999999999998,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.85899999999999999,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.66600000000000004}]},{"osd":6,"last update":"Fri Apr 26 04:53:58 2024","interfaces":[{"interface":"back","average":{"1min":1.6379999999999999,"5min":1.6379999999999999,"15min":1.6379999999999999},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":18.221,"5min":18.221,"15min":18.221},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.57299999999999995}]},{"osd":7,"last update":"Fri Apr 26 04:54:26 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"min":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.64700000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986210,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297984,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447168,"statfs":{"total":95995035648,"available":95689900032,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:54:15 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.499,"15min":0.49299999999999999},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.58499999999999996}]},{"osd":1,"last update":"Fri Apr 26 04:54:15 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.50600000000000001,"15min":0.5},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.56100000000000005}]},{"osd":2,"last update":"Fri Apr 26 04:54:15 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.52900000000000003,"15min":0.53100000000000003},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.70599999999999996,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.50600000000000001,"15min":0.501},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.51900000000000002}]},{"osd":3,"last update":"Fri Apr 26 04:54:15 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.54500000000000004,"15min":0.54000000000000004},"min":{"1min":0.40000000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.54400000000000004,"15min":0.53800000000000003},"min":{"1min":0.38700000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.56999999999999995}]},{"osd":5,"last update":"Fri Apr 26 04:54:42 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.39800000000000002,"15min":0.38600000000000001},"min":{"1min":0.27200000000000002,"5min":0.221,"15min":0.221},"max":{"1min":0.72199999999999998,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.40699999999999997,"15min":0.39300000000000002},"min":{"1min":0.35799999999999998,"5min":0.251,"15min":0.251},"max":{"1min":0.66700000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.47099999999999997}]},{"osd":6,"last update":"Fri Apr 26 04:54:02 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.49199999999999999}]},{"osd":7,"last update":"Fri Apr 26 04:54:21 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.50700000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724574,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":634880,"data_stored":265719,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:54:37 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.54700000000000004,"15min":0.55200000000000005},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.70699999999999996,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.55500000000000005,"15min":0.55900000000000005},"min":{"1min":0.36699999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.71499999999999997,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.45400000000000001}]},{"osd":1,"last update":"Fri Apr 26 04:54:37 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.54700000000000004,"15min":0.55200000000000005},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.55900000000000005,"15min":0.56499999999999995},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.71499999999999997,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.40300000000000002}]},{"osd":2,"last update":"Fri Apr 26 04:54:37 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.55800000000000005,"15min":0.56399999999999995},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.77000000000000002,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.58499999999999996,"15min":0.59399999999999997},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.50700000000000001}]},{"osd":3,"last update":"Fri Apr 26 04:54:37 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.61599999999999999,"15min":0.624},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.77400000000000002,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.60499999999999998,"15min":0.60899999999999999},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.42299999999999999}]},{"osd":4,"last update":"Fri Apr 26 04:54:37 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.40000000000000002,"15min":0.38800000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.41499999999999998,"15min":0.39900000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.438}]},{"osd":6,"last update":"Fri Apr 26 04:53:58 2024","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.5}]},{"osd":7,"last update":"Fri Apr 26 04:54:24 2024","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.49199999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-26T04:54:58.521 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-26T04:54:58.521 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-26T04:54:58.521 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-26T04:54:58.521 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:54:59.889 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:59 smithi012 ceph-mon[26617]: from='client.24533 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:54:59.889 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:54:59 smithi012 ceph-mon[26617]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:59.889 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:59 smithi012 ceph-mon[37567]: from='client.24533 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:54:59.889 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:54:59 smithi012 ceph-mon[37567]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:59.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:59 smithi116 ceph-mon[29897]: from='client.24533 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:54:59.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:54:59 smithi116 ceph-mon[29897]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:00.333 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:00.459 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:55:00 smithi012 systemd[1]: Stopping Ceph alertmanager.a for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:55:01.657 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:01 smithi012 ceph-mon[26617]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:01.658 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:01 smithi012 ceph-mon[37567]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:01.658 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:55:01 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[66283]: ts=2024-04-26T04:55:01.550Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-26T04:55:01.986 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:01 smithi116 ceph-mon[29897]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:02.012 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:55:01 smithi012 podman[70561]: 2024-04-26 04:55:01.750455383 +0000 UTC m=+0.707443627 container died 1a5d42cbab7d497cd40de05c51cf02089025b4b199f8753abbf92a360cfac805 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:55:02.468 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:55:02 smithi012 podman[70561]: 2024-04-26 04:55:02.094781449 +0000 UTC m=+1.051769693 container cleanup 1a5d42cbab7d497cd40de05c51cf02089025b4b199f8753abbf92a360cfac805 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:55:02.469 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:55:02 smithi012 bash[70561]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a 2024-04-26T04:55:02.565 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:55:02.566 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-26T04:55:02.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:02 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1665429852' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:55:02.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:02 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/1665429852' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:55:02.763 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:55:02 smithi012 podman[70598]: 2024-04-26 04:55:02.54760251 +0000 UTC m=+0.970987412 container remove 1a5d42cbab7d497cd40de05c51cf02089025b4b199f8753abbf92a360cfac805 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:55:02.763 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:55:02 smithi012 podman[70598]: 2024-04-26 04:55:02.640310296 +0000 UTC m=+1.063695198 volume remove fa8621803054bc2b14da4a9624a776ff9fe95520e05c452d52e5f167fc68f30f 2024-04-26T04:55:02.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:02 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/1665429852' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:55:03.206 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-26T04:55:03.206 INFO:tasks.cephadm:Setup complete, yielding 2024-04-26T04:55:03.206 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-26T04:55:03.218 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi012.front.sepia.ceph.com 2024-04-26T04:55:03.218 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- bash -c 'ceph fs volume create foo' 2024-04-26T04:55:03.512 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:55:03 smithi012 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-26T04:55:03.513 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:55:03 smithi012 systemd[1]: Stopped Ceph alertmanager.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:55:03.513 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:55:03 smithi012 systemd[1]: Starting Ceph alertmanager.a for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:55:03.818 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:03 smithi012 ceph-mon[26617]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:03.818 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:03 smithi012 ceph-mon[37567]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:03.818 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:55:03 smithi012 podman[70757]: 2024-04-26 04:55:03.446096541 +0000 UTC m=+0.020551070 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-04-26T04:55:03.819 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:55:03 smithi012 podman[70757]: 2024-04-26 04:55:03.553562226 +0000 UTC m=+0.128016775 volume create e0544763e9393ad9e11fd75b51101139cda9af1f0c1b5dd32e248495fb7c1bad 2024-04-26T04:55:03.819 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:55:03 smithi012 podman[70757]: 2024-04-26 04:55:03.637315426 +0000 UTC m=+0.211769933 container create 10573377212bb2428986e5bcef8b447043432313ce97bc6890275f3975b01141 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:55:03.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:03 smithi116 ceph-mon[29897]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:03.992 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:55:03 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:55:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:55:04.141 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:04.258 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:55:03 smithi012 podman[70757]: 2024-04-26 04:55:03.915617727 +0000 UTC m=+0.490072243 container init 10573377212bb2428986e5bcef8b447043432313ce97bc6890275f3975b01141 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:55:04.259 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:55:03 smithi012 podman[70757]: 2024-04-26 04:55:03.918566659 +0000 UTC m=+0.493021167 container start 10573377212bb2428986e5bcef8b447043432313ce97bc6890275f3975b01141 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:55:04.259 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:55:03 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[70793]: ts=2024-04-26T04:55:03.930Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-04-26T04:55:04.259 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:55:03 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[70793]: ts=2024-04-26T04:55:03.930Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-04-26T04:55:04.259 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:55:03 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[70793]: ts=2024-04-26T04:55:03.932Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.12 port=9094 2024-04-26T04:55:04.259 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:55:03 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[70793]: ts=2024-04-26T04:55:03.933Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-26T04:55:04.259 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:55:03 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[70793]: ts=2024-04-26T04:55:03.956Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-26T04:55:04.259 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:55:03 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[70793]: ts=2024-04-26T04:55:03.956Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-26T04:55:04.259 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:55:03 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[70793]: ts=2024-04-26T04:55:03.959Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-04-26T04:55:04.259 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:55:03 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[70793]: ts=2024-04-26T04:55:03.959Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-04-26T04:55:04.259 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:55:03 smithi012 bash[70757]: 10573377212bb2428986e5bcef8b447043432313ce97bc6890275f3975b01141 2024-04-26T04:55:04.259 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:55:04 smithi012 systemd[1]: Started Ceph alertmanager.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:55:05.189 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:05 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:05.189 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:05 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:05.189 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:05 smithi012 ceph-mon[26617]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:55:05.190 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:05 smithi012 ceph-mon[26617]: Reconfiguring daemon prometheus.a on smithi116 2024-04-26T04:55:05.190 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:05 smithi012 ceph-mon[26617]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:05.190 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:05 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:05.190 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:05 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:05.190 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:05 smithi012 ceph-mon[37567]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:55:05.190 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:05 smithi012 ceph-mon[37567]: Reconfiguring daemon prometheus.a on smithi116 2024-04-26T04:55:05.190 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:05 smithi012 ceph-mon[37567]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:05.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:05 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:05.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:05 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:05.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:05 smithi116 ceph-mon[29897]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:55:05.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:05 smithi116 ceph-mon[29897]: Reconfiguring daemon prometheus.a on smithi116 2024-04-26T04:55:05.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:05 smithi116 ceph-mon[29897]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:06.262 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:55:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[70793]: ts=2024-04-26T04:55:05.933Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000314646s 2024-04-26T04:55:06.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:06 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-26T04:55:06.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:06 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-26T04:55:06.762 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:06 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-26T04:55:06.762 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:06 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-26T04:55:06.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:06 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-26T04:55:06.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:06 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-26T04:55:07.220 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:06 smithi116 systemd[1]: Stopping Ceph prometheus.a for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:55:07.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:07 smithi116 ceph-mon[29897]: from='client.24545 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:55:07.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:07 smithi116 ceph-mon[29897]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:07.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:07 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-04-26T04:55:07.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:07 smithi116 ceph-mon[29897]: osdmap e65: 8 total, 8 up, 8 in 2024-04-26T04:55:07.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:07 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-26T04:55:07.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:07 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-26T04:55:07.492 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:07 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[57037]: ts=2024-04-26T04:55:07.219Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-26T04:55:07.492 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:07 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[57037]: ts=2024-04-26T04:55:07.220Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-26T04:55:07.492 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:07 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[57037]: ts=2024-04-26T04:55:07.220Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-26T04:55:07.492 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:07 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[57037]: ts=2024-04-26T04:55:07.220Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-26T04:55:07.492 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:07 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[57037]: ts=2024-04-26T04:55:07.220Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-26T04:55:07.492 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:07 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[57037]: ts=2024-04-26T04:55:07.220Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-26T04:55:07.492 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:07 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[57037]: ts=2024-04-26T04:55:07.220Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-26T04:55:07.492 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:07 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[57037]: ts=2024-04-26T04:55:07.220Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-26T04:55:07.493 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:07 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[57037]: ts=2024-04-26T04:55:07.220Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-26T04:55:07.493 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:07 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[57037]: ts=2024-04-26T04:55:07.220Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-26T04:55:07.493 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:07 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[57037]: ts=2024-04-26T04:55:07.220Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-26T04:55:07.493 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:07 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[57037]: ts=2024-04-26T04:55:07.220Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-26T04:55:07.493 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:07 smithi116 podman[60212]: 2024-04-26 04:55:07.230449246 +0000 UTC m=+0.107627266 container died 36962f3d5221808c45119b01b4c65a2d9816fc2ff4379115fe571bbf592bdd89 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:55:07.762 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:07 smithi012 ceph-mon[37567]: from='client.24545 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:55:07.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:07 smithi012 ceph-mon[37567]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:07.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:07 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-04-26T04:55:07.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:07 smithi012 ceph-mon[37567]: osdmap e65: 8 total, 8 up, 8 in 2024-04-26T04:55:07.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:07 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-26T04:55:07.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:07 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-26T04:55:07.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:07 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-a[26613]: 2024-04-26T04:55:07.450+0000 7fc2c1215700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-26T04:55:07.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:07 smithi012 ceph-mon[26617]: from='client.24545 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:55:07.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:07 smithi012 ceph-mon[26617]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:07.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:07 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-04-26T04:55:07.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:07 smithi012 ceph-mon[26617]: osdmap e65: 8 total, 8 up, 8 in 2024-04-26T04:55:07.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:07 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-26T04:55:07.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:07 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-26T04:55:07.991 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:07 smithi116 podman[60212]: 2024-04-26 04:55:07.69530325 +0000 UTC m=+0.572481279 container cleanup 36962f3d5221808c45119b01b4c65a2d9816fc2ff4379115fe571bbf592bdd89 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:55:07.991 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:07 smithi116 bash[60212]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a 2024-04-26T04:55:08.159 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-04-26T04:55:08.333 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:07 smithi116 podman[60226]: 2024-04-26 04:55:07.996667747 +0000 UTC m=+0.769447815 container remove 36962f3d5221808c45119b01b4c65a2d9816fc2ff4379115fe571bbf592bdd89 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:55:08.334 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:08 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-26T04:55:08.334 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:08 smithi116 systemd[1]: Stopped Ceph prometheus.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:55:08.334 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:08 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 1.025s CPU time. 2024-04-26T04:55:08.334 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:08 smithi116 systemd[1]: Starting Ceph prometheus.a for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:55:08.425 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:08.456 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:08 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-04-26T04:55:08.456 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:08 smithi012 ceph-mon[26617]: osdmap e66: 8 total, 8 up, 8 in 2024-04-26T04:55:08.456 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:08 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-26T04:55:08.456 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:08 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-26T04:55:08.456 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:08 smithi012 ceph-mon[26617]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-26T04:55:08.456 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:08 smithi012 ceph-mon[26617]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-26T04:55:08.456 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:08 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:08.659 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:08 smithi116 podman[60322]: 2024-04-26 04:55:08.237241854 +0000 UTC m=+0.018564163 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-26T04:55:08.659 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:08 smithi116 podman[60322]: 2024-04-26 04:55:08.423008633 +0000 UTC m=+0.204330987 container create db82def51910508c2620309ad320fa7f1b770d2161d6ec43d334d326084f0ded (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:55:08.659 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:08 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-04-26T04:55:08.659 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:08 smithi116 ceph-mon[29897]: osdmap e66: 8 total, 8 up, 8 in 2024-04-26T04:55:08.659 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:08 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-26T04:55:08.660 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:08 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-26T04:55:08.660 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:08 smithi116 ceph-mon[29897]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-26T04:55:08.660 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:08 smithi116 ceph-mon[29897]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-26T04:55:08.660 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:08 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:08.660 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:08 smithi116 ceph-mon[29897]: osdmap e67: 8 total, 8 up, 8 in 2024-04-26T04:55:08.660 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:08 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-04-26T04:55:08.660 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:08 smithi116 ceph-mon[29897]: fsmap foo:0 2024-04-26T04:55:08.660 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:08 smithi116 ceph-mon[29897]: Saving service mds.foo spec with placement count:2 2024-04-26T04:55:08.660 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:08 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:08.724 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:08 smithi012 ceph-mon[26617]: osdmap e67: 8 total, 8 up, 8 in 2024-04-26T04:55:08.724 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:08 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-04-26T04:55:08.724 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:08 smithi012 ceph-mon[26617]: fsmap foo:0 2024-04-26T04:55:08.724 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:08 smithi012 ceph-mon[26617]: Saving service mds.foo spec with placement count:2 2024-04-26T04:55:08.724 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:08 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:08.724 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:08 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-04-26T04:55:08.724 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:08 smithi012 ceph-mon[37567]: osdmap e66: 8 total, 8 up, 8 in 2024-04-26T04:55:08.724 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:08 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-26T04:55:08.725 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:08 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-26T04:55:08.725 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:08 smithi012 ceph-mon[37567]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-26T04:55:08.725 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:08 smithi012 ceph-mon[37567]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-26T04:55:08.725 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:08 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:08.725 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:08 smithi012 ceph-mon[37567]: osdmap e67: 8 total, 8 up, 8 in 2024-04-26T04:55:08.725 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:08 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-04-26T04:55:08.725 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:08 smithi012 ceph-mon[37567]: fsmap foo:0 2024-04-26T04:55:08.725 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:08 smithi012 ceph-mon[37567]: Saving service mds.foo spec with placement count:2 2024-04-26T04:55:08.725 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:08 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:08.992 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:08 smithi116 podman[60322]: 2024-04-26 04:55:08.659279256 +0000 UTC m=+0.440601551 container init db82def51910508c2620309ad320fa7f1b770d2161d6ec43d334d326084f0ded (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:55:08.992 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:08 smithi116 podman[60322]: 2024-04-26 04:55:08.662700275 +0000 UTC m=+0.444022571 container start db82def51910508c2620309ad320fa7f1b770d2161d6ec43d334d326084f0ded (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:55:08.992 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:08 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[60337]: ts=2024-04-26T04:55:08.686Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-26T04:55:08.992 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:08 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[60337]: ts=2024-04-26T04:55:08.686Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-26T04:55:08.992 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:08 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[60337]: ts=2024-04-26T04:55:08.686Z caller=main.go:570 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi116 (none))" 2024-04-26T04:55:08.992 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:08 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[60337]: ts=2024-04-26T04:55:08.686Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-26T04:55:08.992 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:08 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[60337]: ts=2024-04-26T04:55:08.686Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-26T04:55:08.993 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:08 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[60337]: ts=2024-04-26T04:55:08.687Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-26T04:55:08.993 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:08 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[60337]: ts=2024-04-26T04:55:08.688Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-26T04:55:08.993 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:08 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[60337]: ts=2024-04-26T04:55:08.690Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-26T04:55:08.993 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:08 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[60337]: ts=2024-04-26T04:55:08.690Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-26T04:55:08.993 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:08 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[60337]: ts=2024-04-26T04:55:08.691Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-26T04:55:08.993 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:08 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[60337]: ts=2024-04-26T04:55:08.691Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.99µs 2024-04-26T04:55:08.993 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:08 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[60337]: ts=2024-04-26T04:55:08.691Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-26T04:55:08.993 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:08 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[60337]: ts=2024-04-26T04:55:08.691Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-04-26T04:55:08.993 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:08 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[60337]: ts=2024-04-26T04:55:08.691Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-04-26T04:55:08.993 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:08 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[60337]: ts=2024-04-26T04:55:08.691Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=27.801µs wal_replay_duration=482.716µs wbl_replay_duration=112ns total_replay_duration=538.095µs 2024-04-26T04:55:08.993 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:08 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[60337]: ts=2024-04-26T04:55:08.692Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-26T04:55:08.993 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:08 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[60337]: ts=2024-04-26T04:55:08.692Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-26T04:55:08.993 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:08 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[60337]: ts=2024-04-26T04:55:08.692Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-26T04:55:08.993 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:08 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[60337]: ts=2024-04-26T04:55:08.706Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.186143ms db_storage=742ns remote_storage=1.071µs web_handler=263ns query_engine=615ns scrape=381.73µs scrape_sd=27.169µs notify=17.848µs notify_sd=5.26µs rules=13.557198ms tracing=6.141µs 2024-04-26T04:55:08.994 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:08 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[60337]: ts=2024-04-26T04:55:08.706Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-26T04:55:08.994 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:08 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[60337]: ts=2024-04-26T04:55:08.706Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-26T04:55:08.994 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:08 smithi116 bash[60322]: db82def51910508c2620309ad320fa7f1b770d2161d6ec43d334d326084f0ded 2024-04-26T04:55:08.994 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:55:08 smithi116 systemd[1]: Started Ceph prometheus.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:55:09.013 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:55:08 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: [26/Apr/2024:04:55:08] ENGINE Bus STOPPING 2024-04-26T04:55:09.013 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:55:08 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: [26/Apr/2024:04:55:08] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:55:09.013 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:55:08 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: [26/Apr/2024:04:55:08] ENGINE Bus STOPPED 2024-04-26T04:55:09.013 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:55:08 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: [26/Apr/2024:04:55:08] ENGINE Bus STARTING 2024-04-26T04:55:09.013 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:55:08 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: [26/Apr/2024:04:55:08] ENGINE Serving on http://:::9283 2024-04-26T04:55:09.013 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:55:08 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: [26/Apr/2024:04:55:08] ENGINE Bus STARTED 2024-04-26T04:55:09.013 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:55:08 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: [26/Apr/2024:04:55:08] ENGINE Bus STOPPING 2024-04-26T04:55:09.013 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:55:08 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: [26/Apr/2024:04:55:08] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:55:09.013 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:55:08 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: [26/Apr/2024:04:55:08] ENGINE Bus STOPPED 2024-04-26T04:55:09.014 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:55:08 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: [26/Apr/2024:04:55:08] ENGINE Bus STARTING 2024-04-26T04:55:09.468 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:55:09 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: [26/Apr/2024:04:55:09] ENGINE Serving on http://:::9283 2024-04-26T04:55:09.469 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:55:09 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: [26/Apr/2024:04:55:09] ENGINE Bus STARTED 2024-04-26T04:55:09.469 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:55:09 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: [26/Apr/2024:04:55:09] ENGINE Bus STOPPING 2024-04-26T04:55:09.469 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:55:09 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: [26/Apr/2024:04:55:09] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:55:09.469 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:55:09 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: [26/Apr/2024:04:55:09] ENGINE Bus STOPPED 2024-04-26T04:55:09.469 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:55:09 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: [26/Apr/2024:04:55:09] ENGINE Bus STARTING 2024-04-26T04:55:09.469 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:55:09 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: [26/Apr/2024:04:55:09] ENGINE Serving on http://:::9283 2024-04-26T04:55:09.469 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:55:09 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: [26/Apr/2024:04:55:09] ENGINE Bus STARTED 2024-04-26T04:55:09.728 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:09 smithi012 ceph-mon[26617]: pgmap v48: 73 pgs: 17 creating+peering, 56 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:09.728 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:09 smithi012 ceph-mon[26617]: osdmap e68: 8 total, 8 up, 8 in 2024-04-26T04:55:09.728 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:09 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:09.728 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:09 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:09.728 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:09 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:55:09.729 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:09 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi012:9093"}]: dispatch 2024-04-26T04:55:09.729 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:09 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:09.729 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:09 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:55:09.729 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:09 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi116.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:55:09.729 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:09 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:09.729 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:09 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:55:09.729 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:09 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi116.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T04:55:09.729 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:09 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:09.729 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:09 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:55:09.729 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:09 smithi012 ceph-mon[37567]: pgmap v48: 73 pgs: 17 creating+peering, 56 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:09.730 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:09 smithi012 ceph-mon[37567]: osdmap e68: 8 total, 8 up, 8 in 2024-04-26T04:55:09.730 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:09 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:09.730 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:09 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:09.730 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:09 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:55:09.730 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:09 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi012:9093"}]: dispatch 2024-04-26T04:55:09.730 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:09 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:09.730 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:09 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:55:09.730 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:09 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi116.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:55:09.730 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:09 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:09.730 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:09 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:55:09.730 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:09 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi116.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T04:55:09.731 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:09 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:09.731 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:09 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:55:09.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:09 smithi116 ceph-mon[29897]: pgmap v48: 73 pgs: 17 creating+peering, 56 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:09.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:09 smithi116 ceph-mon[29897]: osdmap e68: 8 total, 8 up, 8 in 2024-04-26T04:55:09.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:09 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:09.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:09 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:09.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:09 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:55:09.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:09 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi012:9093"}]: dispatch 2024-04-26T04:55:09.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:09 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:09.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:09 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:55:09.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:09 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi116.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:55:09.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:09 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:09.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:09 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:55:09.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:09 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi116.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T04:55:09.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:09 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:09.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:09 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:55:10.705 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:10 smithi012 ceph-mon[26617]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:55:10.705 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:10 smithi012 ceph-mon[26617]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi012:9093"}]: dispatch 2024-04-26T04:55:10.705 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:10 smithi012 ceph-mon[26617]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:55:10.705 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:10 smithi012 ceph-mon[26617]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi116.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:55:10.705 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:10 smithi012 ceph-mon[26617]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:55:10.705 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:10 smithi012 ceph-mon[26617]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi116.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T04:55:10.705 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:10 smithi012 ceph-mon[26617]: osdmap e69: 8 total, 8 up, 8 in 2024-04-26T04:55:10.706 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:10 smithi012 ceph-mon[37567]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:55:10.706 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:10 smithi012 ceph-mon[37567]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi012:9093"}]: dispatch 2024-04-26T04:55:10.706 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:10 smithi012 ceph-mon[37567]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:55:10.706 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:10 smithi012 ceph-mon[37567]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi116.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:55:10.706 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:10 smithi012 ceph-mon[37567]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:55:10.706 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:10 smithi012 ceph-mon[37567]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi116.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T04:55:10.706 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:10 smithi012 ceph-mon[37567]: osdmap e69: 8 total, 8 up, 8 in 2024-04-26T04:55:10.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:10 smithi116 ceph-mon[29897]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:55:10.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:10 smithi116 ceph-mon[29897]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi012:9093"}]: dispatch 2024-04-26T04:55:10.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:10 smithi116 ceph-mon[29897]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:55:10.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:10 smithi116 ceph-mon[29897]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi116.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:55:10.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:10 smithi116 ceph-mon[29897]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:55:10.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:10 smithi116 ceph-mon[29897]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi116.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T04:55:10.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:10 smithi116 ceph-mon[29897]: osdmap e69: 8 total, 8 up, 8 in 2024-04-26T04:55:11.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:11 smithi116 ceph-mon[29897]: pgmap v51: 73 pgs: 17 creating+peering, 56 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:11.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:11 smithi012 ceph-mon[26617]: pgmap v51: 73 pgs: 17 creating+peering, 56 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:11.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:11 smithi012 ceph-mon[37567]: pgmap v51: 73 pgs: 17 creating+peering, 56 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:12.421 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-04-26T04:55:13.137 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:13.445 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:13 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:13.446 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:13 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:13.446 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:13 smithi012 ceph-mon[26617]: pgmap v52: 73 pgs: 17 creating+peering, 56 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:13.446 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:13 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:13.446 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:13 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:13.446 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:13 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:13.446 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:13 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:13.446 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:13 smithi012 ceph-mon[37567]: pgmap v52: 73 pgs: 17 creating+peering, 56 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:13.446 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:13 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:13.446 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:13 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:13.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:13 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:13.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:13 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:13.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:13 smithi116 ceph-mon[29897]: pgmap v52: 73 pgs: 17 creating+peering, 56 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:13.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:13 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:13.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:13 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:14.012 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:55:13 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[70793]: ts=2024-04-26T04:55:13.934Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.001400341s 2024-04-26T04:55:14.409 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:14 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:55:14.409 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:14 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:55:14.409 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:14 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:14.409 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:14 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi012.fdqwmo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:55:14.409 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:14 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi012.fdqwmo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:55:14.409 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:14 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi012.fdqwmo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-26T04:55:14.410 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:14 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:55:14.410 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:14 smithi012 ceph-mon[37567]: Deploying daemon mds.foo.smithi012.fdqwmo on smithi012 2024-04-26T04:55:14.410 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:14 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:55:14.410 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:14 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:55:14.410 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:14 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:14.410 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:14 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi012.fdqwmo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:55:14.410 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:14 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi012.fdqwmo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:55:14.410 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:14 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi012.fdqwmo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-26T04:55:14.410 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:14 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:55:14.411 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:14 smithi012 ceph-mon[26617]: Deploying daemon mds.foo.smithi012.fdqwmo on smithi012 2024-04-26T04:55:14.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:14 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:55:14.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:14 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:55:14.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:14 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:14.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:14 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi012.fdqwmo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:55:14.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:14 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi012.fdqwmo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:55:14.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:14 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi012.fdqwmo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-26T04:55:14.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:14 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:55:14.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:14 smithi116 ceph-mon[29897]: Deploying daemon mds.foo.smithi012.fdqwmo on smithi012 2024-04-26T04:55:15.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:15 smithi012 ceph-mon[26617]: pgmap v53: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:15.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:15 smithi012 ceph-mon[37567]: pgmap v53: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:15.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:15 smithi116 ceph-mon[29897]: pgmap v53: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:16.785 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-04-26T04:55:16.795 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-26T04:55:16.795 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-26T04:55:16.795 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph mgr dump --format=json 2024-04-26T04:55:17.390 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:17.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:17 smithi116 ceph-mon[29897]: pgmap v54: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:17.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:17 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:17.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:17 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:17.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:17 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:17.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:17 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi116.kduyef", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:55:17.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:17 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi116.kduyef", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:55:17.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:17 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi116.kduyef", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-26T04:55:17.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:17 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:55:17.762 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:17 smithi012 ceph-mon[37567]: pgmap v54: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:17.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:17 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:17.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:17 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:17.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:17 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:17.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:17 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi116.kduyef", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:55:17.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:17 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi116.kduyef", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:55:17.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:17 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi116.kduyef", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-26T04:55:17.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:17 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:55:17.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:17 smithi012 ceph-mon[26617]: pgmap v54: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:17.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:17 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:17.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:17 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:17.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:17 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:17.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:17 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi116.kduyef", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:55:17.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:17 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi116.kduyef", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:55:17.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:17 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi116.kduyef", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-26T04:55:17.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:17 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:55:18.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:18 smithi116 ceph-mon[29897]: Deploying daemon mds.foo.smithi116.kduyef on smithi116 2024-04-26T04:55:18.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:18 smithi116 ceph-mon[29897]: mds.? [v2:172.21.15.12:6834/963323745,v1:172.21.15.12:6835/963323745] up:boot 2024-04-26T04:55:18.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:18 smithi116 ceph-mon[29897]: daemon mds.foo.smithi012.fdqwmo assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-04-26T04:55:18.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:18 smithi116 ceph-mon[29897]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-26T04:55:18.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:18 smithi116 ceph-mon[29897]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-26T04:55:18.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:18 smithi116 ceph-mon[29897]: Cluster is now healthy 2024-04-26T04:55:18.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:18 smithi116 ceph-mon[29897]: fsmap foo:0 1 up:standby 2024-04-26T04:55:18.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:18 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi012.fdqwmo"}]: dispatch 2024-04-26T04:55:18.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:18 smithi116 ceph-mon[29897]: fsmap foo:1 {0=foo.smithi012.fdqwmo=up:creating} 2024-04-26T04:55:18.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:18 smithi116 ceph-mon[29897]: daemon mds.foo.smithi012.fdqwmo is now active in filesystem foo as rank 0 2024-04-26T04:55:18.752 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:18 smithi012 ceph-mon[26617]: Deploying daemon mds.foo.smithi116.kduyef on smithi116 2024-04-26T04:55:18.752 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:18 smithi012 ceph-mon[26617]: mds.? [v2:172.21.15.12:6834/963323745,v1:172.21.15.12:6835/963323745] up:boot 2024-04-26T04:55:18.752 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:18 smithi012 ceph-mon[26617]: daemon mds.foo.smithi012.fdqwmo assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-04-26T04:55:18.752 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:18 smithi012 ceph-mon[26617]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-26T04:55:18.752 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:18 smithi012 ceph-mon[26617]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-26T04:55:18.752 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:18 smithi012 ceph-mon[26617]: Cluster is now healthy 2024-04-26T04:55:18.752 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:18 smithi012 ceph-mon[26617]: fsmap foo:0 1 up:standby 2024-04-26T04:55:18.752 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:18 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi012.fdqwmo"}]: dispatch 2024-04-26T04:55:18.752 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:18 smithi012 ceph-mon[26617]: fsmap foo:1 {0=foo.smithi012.fdqwmo=up:creating} 2024-04-26T04:55:18.752 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:18 smithi012 ceph-mon[26617]: daemon mds.foo.smithi012.fdqwmo is now active in filesystem foo as rank 0 2024-04-26T04:55:18.753 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:18 smithi012 ceph-mon[37567]: Deploying daemon mds.foo.smithi116.kduyef on smithi116 2024-04-26T04:55:18.753 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:18 smithi012 ceph-mon[37567]: mds.? [v2:172.21.15.12:6834/963323745,v1:172.21.15.12:6835/963323745] up:boot 2024-04-26T04:55:18.753 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:18 smithi012 ceph-mon[37567]: daemon mds.foo.smithi012.fdqwmo assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-04-26T04:55:18.753 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:18 smithi012 ceph-mon[37567]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-26T04:55:18.753 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:18 smithi012 ceph-mon[37567]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-26T04:55:18.753 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:18 smithi012 ceph-mon[37567]: Cluster is now healthy 2024-04-26T04:55:18.753 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:18 smithi012 ceph-mon[37567]: fsmap foo:0 1 up:standby 2024-04-26T04:55:18.753 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:18 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi012.fdqwmo"}]: dispatch 2024-04-26T04:55:18.753 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:18 smithi012 ceph-mon[37567]: fsmap foo:1 {0=foo.smithi012.fdqwmo=up:creating} 2024-04-26T04:55:18.753 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:18 smithi012 ceph-mon[37567]: daemon mds.foo.smithi012.fdqwmo is now active in filesystem foo as rank 0 2024-04-26T04:55:19.350 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:55:19.644 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:19 smithi012 ceph-mon[26617]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-04-26T04:55:19.644 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:19 smithi012 ceph-mon[26617]: mds.? [v2:172.21.15.12:6834/963323745,v1:172.21.15.12:6835/963323745] up:active 2024-04-26T04:55:19.644 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:19 smithi012 ceph-mon[26617]: fsmap foo:1 {0=foo.smithi012.fdqwmo=up:active} 2024-04-26T04:55:19.644 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:19 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/3297625455' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T04:55:19.644 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:19 smithi012 ceph-mon[37567]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-04-26T04:55:19.644 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:19 smithi012 ceph-mon[37567]: mds.? [v2:172.21.15.12:6834/963323745,v1:172.21.15.12:6835/963323745] up:active 2024-04-26T04:55:19.644 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:19 smithi012 ceph-mon[37567]: fsmap foo:1 {0=foo.smithi012.fdqwmo=up:active} 2024-04-26T04:55:19.644 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:19 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/3297625455' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T04:55:19.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:19 smithi116 ceph-mon[29897]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-04-26T04:55:19.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:19 smithi116 ceph-mon[29897]: mds.? [v2:172.21.15.12:6834/963323745,v1:172.21.15.12:6835/963323745] up:active 2024-04-26T04:55:19.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:19 smithi116 ceph-mon[29897]: fsmap foo:1 {0=foo.smithi012.fdqwmo=up:active} 2024-04-26T04:55:19.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:19 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/3297625455' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T04:55:20.003 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":20,"flags":0,"active_gid":14454,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6800","nonce":2470533793},{"type":"v1","addr":"172.21.15.12:6801","nonce":2470533793}]},"active_addr":"172.21.15.12:6801/2470533793","active_change":"2024-04-26T04:53:52.395055+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24334,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.12:8443/","prometheus":"http://172.21.15.12:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.12:0","nonce":1826681709}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.12:0","nonce":3390053105}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.12:0","nonce":2567163031}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.12:0","nonce":2417119692}]}]} 2024-04-26T04:55:20.008 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-26T04:55:20.008 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-26T04:55:20.008 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T04:55:20.292 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:20.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:20 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:20.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:20 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:20.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:20 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:20.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:20 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:20.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:20 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:20.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:20 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:55:20.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:20 smithi116 ceph-mon[29897]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-04-26T04:55:20.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:20 smithi116 ceph-mon[29897]: mds.? [v2:172.21.15.116:6832/529792701,v1:172.21.15.116:6833/529792701] up:boot 2024-04-26T04:55:20.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:20 smithi116 ceph-mon[29897]: fsmap foo:1 {0=foo.smithi012.fdqwmo=up:active} 1 up:standby 2024-04-26T04:55:20.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:20 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi116.kduyef"}]: dispatch 2024-04-26T04:55:20.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:20 smithi116 ceph-mon[29897]: fsmap foo:1 {0=foo.smithi012.fdqwmo=up:active} 1 up:standby 2024-04-26T04:55:21.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:20 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:21.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:20 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:21.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:20 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:21.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:20 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:21.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:20 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:21.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:20 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:55:21.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:20 smithi012 ceph-mon[26617]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-04-26T04:55:21.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:20 smithi012 ceph-mon[26617]: mds.? [v2:172.21.15.116:6832/529792701,v1:172.21.15.116:6833/529792701] up:boot 2024-04-26T04:55:21.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:20 smithi012 ceph-mon[26617]: fsmap foo:1 {0=foo.smithi012.fdqwmo=up:active} 1 up:standby 2024-04-26T04:55:21.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:20 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi116.kduyef"}]: dispatch 2024-04-26T04:55:21.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:20 smithi012 ceph-mon[26617]: fsmap foo:1 {0=foo.smithi012.fdqwmo=up:active} 1 up:standby 2024-04-26T04:55:21.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:20 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:21.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:20 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:21.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:20 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:21.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:20 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:21.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:20 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:21.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:20 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:55:21.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:20 smithi012 ceph-mon[37567]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-04-26T04:55:21.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:20 smithi012 ceph-mon[37567]: mds.? [v2:172.21.15.116:6832/529792701,v1:172.21.15.116:6833/529792701] up:boot 2024-04-26T04:55:21.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:20 smithi012 ceph-mon[37567]: fsmap foo:1 {0=foo.smithi012.fdqwmo=up:active} 1 up:standby 2024-04-26T04:55:21.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:20 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi116.kduyef"}]: dispatch 2024-04-26T04:55:21.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:20 smithi012 ceph-mon[37567]: fsmap foo:1 {0=foo.smithi012.fdqwmo=up:active} 1 up:standby 2024-04-26T04:55:23.015 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:55:23.016 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:22 smithi012 ceph-mon[26617]: mds.? [v2:172.21.15.12:6834/963323745,v1:172.21.15.12:6835/963323745] up:active 2024-04-26T04:55:23.016 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:22 smithi012 ceph-mon[26617]: fsmap foo:1 {0=foo.smithi012.fdqwmo=up:active} 1 up:standby 2024-04-26T04:55:23.016 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:22 smithi012 ceph-mon[26617]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:55:23.016 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:22 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:23.016 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:22 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:23.016 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:55:22 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:55:22] "GET /metrics HTTP/1.1" 200 239083 "" "Prometheus/2.43.0" 2024-04-26T04:55:23.017 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:22 smithi012 ceph-mon[37567]: mds.? [v2:172.21.15.12:6834/963323745,v1:172.21.15.12:6835/963323745] up:active 2024-04-26T04:55:23.017 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:22 smithi012 ceph-mon[37567]: fsmap foo:1 {0=foo.smithi012.fdqwmo=up:active} 1 up:standby 2024-04-26T04:55:23.017 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:22 smithi012 ceph-mon[37567]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:55:23.017 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:22 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:23.017 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:22 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:23.017 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":69,"fsid":"0b9350c6-0388-11ef-bc93-c7b262605968","created":"2024-04-26T04:48:17.966774+0000","modified":"2024-04-26T04:55:09.467486+0000","last_up_change":"2024-04-26T04:53:21.501626+0000","last_in_change":"2024-04-26T04:53:04.660871+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:51:27.066997+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-26T04:53:56.453197+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-26T04:55:06.383315+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-26T04:55:06.882235+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"5f43d5c7-23d9-48a5-bc3c-a3a6ce3c004f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6802","nonce":1077619568},{"type":"v1","addr":"172.21.15.12:6803","nonce":1077619568}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6804","nonce":1077619568},{"type":"v1","addr":"172.21.15.12:6805","nonce":1077619568}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6808","nonce":1077619568},{"type":"v1","addr":"172.21.15.12:6809","nonce":1077619568}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6806","nonce":1077619568},{"type":"v1","addr":"172.21.15.12:6807","nonce":1077619568}]},"public_addr":"172.21.15.12:6803/1077619568","cluster_addr":"172.21.15.12:6805/1077619568","heartbeat_back_addr":"172.21.15.12:6809/1077619568","heartbeat_front_addr":"172.21.15.12:6807/1077619568","state":["exists","up"]},{"osd":1,"uuid":"75471714-6e29-4e02-9c03-86ce919b1ff0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6810","nonce":726116340},{"type":"v1","addr":"172.21.15.12:6811","nonce":726116340}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6812","nonce":726116340},{"type":"v1","addr":"172.21.15.12:6813","nonce":726116340}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6816","nonce":726116340},{"type":"v1","addr":"172.21.15.12:6817","nonce":726116340}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6814","nonce":726116340},{"type":"v1","addr":"172.21.15.12:6815","nonce":726116340}]},"public_addr":"172.21.15.12:6811/726116340","cluster_addr":"172.21.15.12:6813/726116340","heartbeat_back_addr":"172.21.15.12:6817/726116340","heartbeat_front_addr":"172.21.15.12:6815/726116340","state":["exists","up"]},{"osd":2,"uuid":"4b402201-4f28-4bc1-9cf0-0384090fb158","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6818","nonce":4118496923},{"type":"v1","addr":"172.21.15.12:6819","nonce":4118496923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6820","nonce":4118496923},{"type":"v1","addr":"172.21.15.12:6821","nonce":4118496923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6824","nonce":4118496923},{"type":"v1","addr":"172.21.15.12:6825","nonce":4118496923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6822","nonce":4118496923},{"type":"v1","addr":"172.21.15.12:6823","nonce":4118496923}]},"public_addr":"172.21.15.12:6819/4118496923","cluster_addr":"172.21.15.12:6821/4118496923","heartbeat_back_addr":"172.21.15.12:6825/4118496923","heartbeat_front_addr":"172.21.15.12:6823/4118496923","state":["exists","up"]},{"osd":3,"uuid":"54bf03b9-b2e6-4230-8dcd-d62af8853c36","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6826","nonce":3374180434},{"type":"v1","addr":"172.21.15.12:6827","nonce":3374180434}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6828","nonce":3374180434},{"type":"v1","addr":"172.21.15.12:6829","nonce":3374180434}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6832","nonce":3374180434},{"type":"v1","addr":"172.21.15.12:6833","nonce":3374180434}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6830","nonce":3374180434},{"type":"v1","addr":"172.21.15.12:6831","nonce":3374180434}]},"public_addr":"172.21.15.12:6827/3374180434","cluster_addr":"172.21.15.12:6829/3374180434","heartbeat_back_addr":"172.21.15.12:6833/3374180434","heartbeat_front_addr":"172.21.15.12:6831/3374180434","state":["exists","up"]},{"osd":4,"uuid":"ea9d3fb5-1877-4971-a8fb-8b5aafa7871a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6800","nonce":2011782592},{"type":"v1","addr":"172.21.15.116:6801","nonce":2011782592}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6802","nonce":2011782592},{"type":"v1","addr":"172.21.15.116:6803","nonce":2011782592}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6806","nonce":2011782592},{"type":"v1","addr":"172.21.15.116:6807","nonce":2011782592}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6804","nonce":2011782592},{"type":"v1","addr":"172.21.15.116:6805","nonce":2011782592}]},"public_addr":"172.21.15.116:6801/2011782592","cluster_addr":"172.21.15.116:6803/2011782592","heartbeat_back_addr":"172.21.15.116:6807/2011782592","heartbeat_front_addr":"172.21.15.116:6805/2011782592","state":["exists","up"]},{"osd":5,"uuid":"59f0530d-37eb-41d8-86d0-580ba9b8e4e2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6808","nonce":4077435109},{"type":"v1","addr":"172.21.15.116:6809","nonce":4077435109}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6810","nonce":4077435109},{"type":"v1","addr":"172.21.15.116:6811","nonce":4077435109}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6814","nonce":4077435109},{"type":"v1","addr":"172.21.15.116:6815","nonce":4077435109}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6812","nonce":4077435109},{"type":"v1","addr":"172.21.15.116:6813","nonce":4077435109}]},"public_addr":"172.21.15.116:6809/4077435109","cluster_addr":"172.21.15.116:6811/4077435109","heartbeat_back_addr":"172.21.15.116:6815/4077435109","heartbeat_front_addr":"172.21.15.116:6813/4077435109","state":["exists","up"]},{"osd":6,"uuid":"62b1edf5-75b9-4ea6-af34-d40584a6c343","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6816","nonce":2560073849},{"type":"v1","addr":"172.21.15.116:6817","nonce":2560073849}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6818","nonce":2560073849},{"type":"v1","addr":"172.21.15.116:6819","nonce":2560073849}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6822","nonce":2560073849},{"type":"v1","addr":"172.21.15.116:6823","nonce":2560073849}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6820","nonce":2560073849},{"type":"v1","addr":"172.21.15.116:6821","nonce":2560073849}]},"public_addr":"172.21.15.116:6817/2560073849","cluster_addr":"172.21.15.116:6819/2560073849","heartbeat_back_addr":"172.21.15.116:6823/2560073849","heartbeat_front_addr":"172.21.15.116:6821/2560073849","state":["exists","up"]},{"osd":7,"uuid":"439d336e-0012-4b61-893f-50f562863965","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6824","nonce":3779511167},{"type":"v1","addr":"172.21.15.116:6825","nonce":3779511167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6826","nonce":3779511167},{"type":"v1","addr":"172.21.15.116:6827","nonce":3779511167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6830","nonce":3779511167},{"type":"v1","addr":"172.21.15.116:6831","nonce":3779511167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6828","nonce":3779511167},{"type":"v1","addr":"172.21.15.116:6829","nonce":3779511167}]},"public_addr":"172.21.15.116:6825/3779511167","cluster_addr":"172.21.15.116:6827/3779511167","heartbeat_back_addr":"172.21.15.116:6831/3779511167","heartbeat_front_addr":"172.21.15.116:6829/3779511167","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:50:34.985771+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:50:58.677899+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:51:23.046413+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:51:48.496821+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:09.812658+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:32.853940+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:56.215123+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:53:19.840326+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.12:0/625629550":"2024-04-27T04:53:52.394847+0000","172.21.15.12:0/824190310":"2024-04-27T04:53:52.394847+0000","172.21.15.12:0/2421619470":"2024-04-27T04:53:52.394847+0000","172.21.15.12:6801/1235528000":"2024-04-27T04:53:52.394847+0000","172.21.15.12:0/4212767587":"2024-04-27T04:49:16.560212+0000","172.21.15.12:0/1150032450":"2024-04-27T04:49:16.560212+0000","172.21.15.12:6801/2681852455":"2024-04-27T04:49:16.560212+0000","172.21.15.12:6800/1235528000":"2024-04-27T04:53:52.394847+0000","172.21.15.12:6801/378740862":"2024-04-27T04:48:52.473898+0000","172.21.15.12:6800/378740862":"2024-04-27T04:48:52.473898+0000","172.21.15.12:0/320935135":"2024-04-27T04:48:52.473898+0000","172.21.15.12:0/4007370348":"2024-04-27T04:49:16.560212+0000","172.21.15.12:0/2016967349":"2024-04-27T04:48:52.473898+0000","172.21.15.12:0/2418603751":"2024-04-27T04:48:52.473898+0000","172.21.15.12:0/1087977851":"2024-04-27T04:53:52.394847+0000","172.21.15.12:6800/2681852455":"2024-04-27T04:49:16.560212+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T04:55:23.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:22 smithi116 ceph-mon[29897]: mds.? [v2:172.21.15.12:6834/963323745,v1:172.21.15.12:6835/963323745] up:active 2024-04-26T04:55:23.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:22 smithi116 ceph-mon[29897]: fsmap foo:1 {0=foo.smithi012.fdqwmo=up:active} 1 up:standby 2024-04-26T04:55:23.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:22 smithi116 ceph-mon[29897]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:55:23.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:22 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:23.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:22 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:23.718 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-26T04:55:23.718 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T04:55:23.881 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:55:23 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:55:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:55:23.986 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:23 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/754016833' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:55:23.986 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:23 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:23.986 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:23 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:23.986 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:23 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/754016833' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:55:23.986 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:23 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:23.986 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:23 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:24.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:23 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/754016833' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:55:24.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:23 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:24.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:23 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:25.228 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:25.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:24 smithi116 ceph-mon[29897]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:55:25.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:24 smithi116 ceph-mon[29897]: mds.? [v2:172.21.15.116:6832/529792701,v1:172.21.15.116:6833/529792701] up:standby 2024-04-26T04:55:25.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:24 smithi116 ceph-mon[29897]: fsmap foo:1 {0=foo.smithi012.fdqwmo=up:active} 1 up:standby 2024-04-26T04:55:25.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:24 smithi012 ceph-mon[26617]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:55:25.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:24 smithi012 ceph-mon[26617]: mds.? [v2:172.21.15.116:6832/529792701,v1:172.21.15.116:6833/529792701] up:standby 2024-04-26T04:55:25.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:24 smithi012 ceph-mon[26617]: fsmap foo:1 {0=foo.smithi012.fdqwmo=up:active} 1 up:standby 2024-04-26T04:55:25.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:24 smithi012 ceph-mon[37567]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:55:25.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:24 smithi012 ceph-mon[37567]: mds.? [v2:172.21.15.116:6832/529792701,v1:172.21.15.116:6833/529792701] up:standby 2024-04-26T04:55:25.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:24 smithi012 ceph-mon[37567]: fsmap foo:1 {0=foo.smithi012.fdqwmo=up:active} 1 up:standby 2024-04-26T04:55:26.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:26 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:26.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:26 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:26.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:26 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:55:26.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:26 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:55:26.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:26 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:26.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:26 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:55:26.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:26 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:26.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:26 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:26.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:26 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:55:26.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:26 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:55:26.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:26 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:26.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:26 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:55:26.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:26 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:26.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:26 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:26.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:26 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:55:26.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:26 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:55:26.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:26 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:26.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:26 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:55:27.134 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:55:27.135 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":69,"fsid":"0b9350c6-0388-11ef-bc93-c7b262605968","created":"2024-04-26T04:48:17.966774+0000","modified":"2024-04-26T04:55:09.467486+0000","last_up_change":"2024-04-26T04:53:21.501626+0000","last_in_change":"2024-04-26T04:53:04.660871+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:51:27.066997+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-26T04:53:56.453197+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-26T04:55:06.383315+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-26T04:55:06.882235+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"5f43d5c7-23d9-48a5-bc3c-a3a6ce3c004f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6802","nonce":1077619568},{"type":"v1","addr":"172.21.15.12:6803","nonce":1077619568}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6804","nonce":1077619568},{"type":"v1","addr":"172.21.15.12:6805","nonce":1077619568}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6808","nonce":1077619568},{"type":"v1","addr":"172.21.15.12:6809","nonce":1077619568}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6806","nonce":1077619568},{"type":"v1","addr":"172.21.15.12:6807","nonce":1077619568}]},"public_addr":"172.21.15.12:6803/1077619568","cluster_addr":"172.21.15.12:6805/1077619568","heartbeat_back_addr":"172.21.15.12:6809/1077619568","heartbeat_front_addr":"172.21.15.12:6807/1077619568","state":["exists","up"]},{"osd":1,"uuid":"75471714-6e29-4e02-9c03-86ce919b1ff0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6810","nonce":726116340},{"type":"v1","addr":"172.21.15.12:6811","nonce":726116340}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6812","nonce":726116340},{"type":"v1","addr":"172.21.15.12:6813","nonce":726116340}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6816","nonce":726116340},{"type":"v1","addr":"172.21.15.12:6817","nonce":726116340}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6814","nonce":726116340},{"type":"v1","addr":"172.21.15.12:6815","nonce":726116340}]},"public_addr":"172.21.15.12:6811/726116340","cluster_addr":"172.21.15.12:6813/726116340","heartbeat_back_addr":"172.21.15.12:6817/726116340","heartbeat_front_addr":"172.21.15.12:6815/726116340","state":["exists","up"]},{"osd":2,"uuid":"4b402201-4f28-4bc1-9cf0-0384090fb158","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6818","nonce":4118496923},{"type":"v1","addr":"172.21.15.12:6819","nonce":4118496923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6820","nonce":4118496923},{"type":"v1","addr":"172.21.15.12:6821","nonce":4118496923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6824","nonce":4118496923},{"type":"v1","addr":"172.21.15.12:6825","nonce":4118496923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6822","nonce":4118496923},{"type":"v1","addr":"172.21.15.12:6823","nonce":4118496923}]},"public_addr":"172.21.15.12:6819/4118496923","cluster_addr":"172.21.15.12:6821/4118496923","heartbeat_back_addr":"172.21.15.12:6825/4118496923","heartbeat_front_addr":"172.21.15.12:6823/4118496923","state":["exists","up"]},{"osd":3,"uuid":"54bf03b9-b2e6-4230-8dcd-d62af8853c36","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6826","nonce":3374180434},{"type":"v1","addr":"172.21.15.12:6827","nonce":3374180434}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6828","nonce":3374180434},{"type":"v1","addr":"172.21.15.12:6829","nonce":3374180434}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6832","nonce":3374180434},{"type":"v1","addr":"172.21.15.12:6833","nonce":3374180434}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6830","nonce":3374180434},{"type":"v1","addr":"172.21.15.12:6831","nonce":3374180434}]},"public_addr":"172.21.15.12:6827/3374180434","cluster_addr":"172.21.15.12:6829/3374180434","heartbeat_back_addr":"172.21.15.12:6833/3374180434","heartbeat_front_addr":"172.21.15.12:6831/3374180434","state":["exists","up"]},{"osd":4,"uuid":"ea9d3fb5-1877-4971-a8fb-8b5aafa7871a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6800","nonce":2011782592},{"type":"v1","addr":"172.21.15.116:6801","nonce":2011782592}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6802","nonce":2011782592},{"type":"v1","addr":"172.21.15.116:6803","nonce":2011782592}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6806","nonce":2011782592},{"type":"v1","addr":"172.21.15.116:6807","nonce":2011782592}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6804","nonce":2011782592},{"type":"v1","addr":"172.21.15.116:6805","nonce":2011782592}]},"public_addr":"172.21.15.116:6801/2011782592","cluster_addr":"172.21.15.116:6803/2011782592","heartbeat_back_addr":"172.21.15.116:6807/2011782592","heartbeat_front_addr":"172.21.15.116:6805/2011782592","state":["exists","up"]},{"osd":5,"uuid":"59f0530d-37eb-41d8-86d0-580ba9b8e4e2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6808","nonce":4077435109},{"type":"v1","addr":"172.21.15.116:6809","nonce":4077435109}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6810","nonce":4077435109},{"type":"v1","addr":"172.21.15.116:6811","nonce":4077435109}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6814","nonce":4077435109},{"type":"v1","addr":"172.21.15.116:6815","nonce":4077435109}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6812","nonce":4077435109},{"type":"v1","addr":"172.21.15.116:6813","nonce":4077435109}]},"public_addr":"172.21.15.116:6809/4077435109","cluster_addr":"172.21.15.116:6811/4077435109","heartbeat_back_addr":"172.21.15.116:6815/4077435109","heartbeat_front_addr":"172.21.15.116:6813/4077435109","state":["exists","up"]},{"osd":6,"uuid":"62b1edf5-75b9-4ea6-af34-d40584a6c343","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6816","nonce":2560073849},{"type":"v1","addr":"172.21.15.116:6817","nonce":2560073849}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6818","nonce":2560073849},{"type":"v1","addr":"172.21.15.116:6819","nonce":2560073849}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6822","nonce":2560073849},{"type":"v1","addr":"172.21.15.116:6823","nonce":2560073849}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6820","nonce":2560073849},{"type":"v1","addr":"172.21.15.116:6821","nonce":2560073849}]},"public_addr":"172.21.15.116:6817/2560073849","cluster_addr":"172.21.15.116:6819/2560073849","heartbeat_back_addr":"172.21.15.116:6823/2560073849","heartbeat_front_addr":"172.21.15.116:6821/2560073849","state":["exists","up"]},{"osd":7,"uuid":"439d336e-0012-4b61-893f-50f562863965","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6824","nonce":3779511167},{"type":"v1","addr":"172.21.15.116:6825","nonce":3779511167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6826","nonce":3779511167},{"type":"v1","addr":"172.21.15.116:6827","nonce":3779511167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6830","nonce":3779511167},{"type":"v1","addr":"172.21.15.116:6831","nonce":3779511167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6828","nonce":3779511167},{"type":"v1","addr":"172.21.15.116:6829","nonce":3779511167}]},"public_addr":"172.21.15.116:6825/3779511167","cluster_addr":"172.21.15.116:6827/3779511167","heartbeat_back_addr":"172.21.15.116:6831/3779511167","heartbeat_front_addr":"172.21.15.116:6829/3779511167","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:50:34.985771+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:50:58.677899+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:51:23.046413+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:51:48.496821+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:09.812658+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:32.853940+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:56.215123+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:53:19.840326+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.12:0/625629550":"2024-04-27T04:53:52.394847+0000","172.21.15.12:0/824190310":"2024-04-27T04:53:52.394847+0000","172.21.15.12:0/2421619470":"2024-04-27T04:53:52.394847+0000","172.21.15.12:6801/1235528000":"2024-04-27T04:53:52.394847+0000","172.21.15.12:0/4212767587":"2024-04-27T04:49:16.560212+0000","172.21.15.12:0/1150032450":"2024-04-27T04:49:16.560212+0000","172.21.15.12:6801/2681852455":"2024-04-27T04:49:16.560212+0000","172.21.15.12:6800/1235528000":"2024-04-27T04:53:52.394847+0000","172.21.15.12:6801/378740862":"2024-04-27T04:48:52.473898+0000","172.21.15.12:6800/378740862":"2024-04-27T04:48:52.473898+0000","172.21.15.12:0/320935135":"2024-04-27T04:48:52.473898+0000","172.21.15.12:0/4007370348":"2024-04-27T04:49:16.560212+0000","172.21.15.12:0/2016967349":"2024-04-27T04:48:52.473898+0000","172.21.15.12:0/2418603751":"2024-04-27T04:48:52.473898+0000","172.21.15.12:0/1087977851":"2024-04-27T04:53:52.394847+0000","172.21.15.12:6800/2681852455":"2024-04-27T04:49:16.560212+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T04:55:27.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:27 smithi116 ceph-mon[29897]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:55:27.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:27 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/3533133794' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:55:27.512 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:27 smithi012 ceph-mon[26617]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:55:27.512 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:27 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/3533133794' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:55:27.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:27 smithi012 ceph-mon[37567]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:55:27.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:27 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/3533133794' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:55:28.005 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-26T04:55:28.005 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-26T04:55:28.006 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-26T04:55:28.006 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-26T04:55:28.007 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-26T04:55:28.007 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-26T04:55:28.007 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-26T04:55:28.008 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-26T04:55:28.639 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:28.643 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:28.644 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:28.644 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:28.648 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:28.648 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:28.651 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:28.651 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:29.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:29 smithi116 ceph-mon[29897]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:55:29.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:29 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:29.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:29 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:29.762 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:29 smithi012 ceph-mon[37567]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:55:29.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:29 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:29.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:29 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:29.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:29 smithi012 ceph-mon[26617]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:55:29.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:29 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:29.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:29 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:31.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:31 smithi116 ceph-mon[29897]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-26T04:55:31.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:31 smithi012 ceph-mon[26617]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-26T04:55:31.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:31 smithi012 ceph-mon[37567]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-26T04:55:33.007 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:55:32 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:55:32] "GET /metrics HTTP/1.1" 200 239083 "" "Prometheus/2.43.0" 2024-04-26T04:55:33.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:33 smithi116 ceph-mon[29897]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-26T04:55:33.742 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:55:33 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:55:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:55:33.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:33 smithi012 ceph-mon[26617]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-26T04:55:33.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:33 smithi012 ceph-mon[37567]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-26T04:55:35.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:35 smithi116 ceph-mon[29897]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-26T04:55:35.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:35 smithi012 ceph-mon[26617]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-26T04:55:35.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:35 smithi012 ceph-mon[37567]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-26T04:55:37.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:37 smithi116 ceph-mon[29897]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:37.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:37 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:37.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:37 smithi012 ceph-mon[26617]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:37.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:37 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:37.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:37 smithi012 ceph-mon[37567]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:37.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:37 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:39.519 INFO:teuthology.orchestra.run.smithi012.stdout:133143986220 2024-04-26T04:55:39.519 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-26T04:55:39.625 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:39 smithi012 ceph-mon[37567]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:39.625 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:39 smithi012 ceph-mon[26617]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:39.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:39 smithi116 ceph-mon[29897]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:40.781 INFO:teuthology.orchestra.run.smithi012.stdout:55834574906 2024-04-26T04:55:40.782 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-26T04:55:41.115 INFO:teuthology.orchestra.run.smithi012.stdout:77309411381 2024-04-26T04:55:41.115 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-26T04:55:41.573 INFO:teuthology.orchestra.run.smithi012.stdout:34359738430 2024-04-26T04:55:41.573 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-26T04:55:41.738 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:41 smithi012 ceph-mon[26617]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:41.738 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:41 smithi012 ceph-mon[37567]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:41.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:41 smithi116 ceph-mon[29897]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:43.012 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:55:42 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:55:42] "GET /metrics HTTP/1.1" 200 239070 "" "Prometheus/2.43.0" 2024-04-26T04:55:43.509 INFO:teuthology.orchestra.run.smithi012.stdout:167503724584 2024-04-26T04:55:43.510 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-26T04:55:43.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:43 smithi012 ceph-mon[26617]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:43.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:43 smithi012 ceph-mon[37567]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:43.991 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:55:43 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:55:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:55:43.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:43 smithi116 ceph-mon[29897]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:45.465 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:45.466 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:45.466 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:45.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:45 smithi012 ceph-mon[26617]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:45.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:45 smithi012 ceph-mon[37567]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:45.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:45 smithi116 ceph-mon[29897]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:46.049 INFO:teuthology.orchestra.run.smithi012.stdout:201863462948 2024-04-26T04:55:46.049 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-26T04:55:47.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:47 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:47.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:47 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:47.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:47 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:55:47.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:47 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:55:47.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:47 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:47.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:47 smithi012 ceph-mon[26617]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:47.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:47 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:47.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:47 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:47.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:47 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:55:47.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:47 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:55:47.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:47 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:47.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:47 smithi012 ceph-mon[37567]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:47.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:47 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:47.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:47 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:47.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:47 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:55:47.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:47 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:55:47.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:47 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:55:47.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:47 smithi116 ceph-mon[29897]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:47.970 INFO:teuthology.orchestra.run.smithi012.stdout:236223201311 2024-04-26T04:55:47.970 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-26T04:55:49.062 INFO:teuthology.orchestra.run.smithi012.stdout:111669149745 2024-04-26T04:55:49.062 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-26T04:55:49.476 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:49.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:49 smithi116 ceph-mon[29897]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:49.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:49 smithi012 ceph-mon[26617]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:49.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:49 smithi012 ceph-mon[37567]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:50.344 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:51.163 INFO:teuthology.orchestra.run.smithi012.stdout:133143986222 2024-04-26T04:55:51.349 INFO:teuthology.orchestra.run.smithi012.stdout:34359738433 2024-04-26T04:55:51.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:51 smithi116 ceph-mon[29897]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:51.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:51 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/2618586496' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T04:55:51.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:51 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/1007038' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T04:55:51.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:51 smithi012 ceph-mon[26617]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:51.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:51 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/2618586496' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T04:55:51.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:51 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1007038' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T04:55:51.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:51 smithi012 ceph-mon[37567]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:51.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:51 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/2618586496' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T04:55:51.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:51 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/1007038' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T04:55:52.394 INFO:teuthology.orchestra.run.smithi012.stdout:77309411384 2024-04-26T04:55:52.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:52 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/532796277' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T04:55:52.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:52 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-04-26T04:55:52.765 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:52 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2024-04-26T04:55:52.765 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:52 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-04-26T04:55:52.765 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:52 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2024-04-26T04:55:52.765 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:52 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2024-04-26T04:55:52.765 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:52 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2024-04-26T04:55:52.765 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:52 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:52.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:52 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/532796277' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T04:55:52.766 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:52 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-04-26T04:55:52.766 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:52 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2024-04-26T04:55:52.766 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:52 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-04-26T04:55:52.766 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:52 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2024-04-26T04:55:52.766 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:52 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2024-04-26T04:55:52.766 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:52 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2024-04-26T04:55:52.766 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:52 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:52.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:52 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/532796277' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T04:55:52.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:52 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-04-26T04:55:52.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:52 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2024-04-26T04:55:52.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:52 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-04-26T04:55:52.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:52 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2024-04-26T04:55:52.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:52 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2024-04-26T04:55:52.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:52 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2024-04-26T04:55:52.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:52 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:53.183 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:55:52 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:55:52] "GET /metrics HTTP/1.1" 200 239045 "" "Prometheus/2.43.0" 2024-04-26T04:55:53.211 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738430 got 34359738433 for osd.0 2024-04-26T04:55:53.211 DEBUG:teuthology.parallel:result is None 2024-04-26T04:55:53.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:53 smithi012 ceph-mon[26617]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:53.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:53 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-04-26T04:55:53.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:53 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-04-26T04:55:53.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:53 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2024-04-26T04:55:53.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:53 smithi012 ceph-mon[26617]: osdmap e70: 8 total, 8 up, 8 in 2024-04-26T04:55:53.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:53 smithi012 ceph-mon[37567]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:53.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:53 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-04-26T04:55:53.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:53 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-04-26T04:55:53.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:53 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2024-04-26T04:55:53.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:53 smithi012 ceph-mon[37567]: osdmap e70: 8 total, 8 up, 8 in 2024-04-26T04:55:53.954 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:53.957 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:53.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:53 smithi116 ceph-mon[29897]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:53.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:53 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-04-26T04:55:53.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:53 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-04-26T04:55:53.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:53 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2024-04-26T04:55:53.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:53 smithi116 ceph-mon[29897]: osdmap e70: 8 total, 8 up, 8 in 2024-04-26T04:55:53.992 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:55:53 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:55:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:55:54.501 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411381 got 77309411384 for osd.2 2024-04-26T04:55:54.501 DEBUG:teuthology.parallel:result is None 2024-04-26T04:55:54.762 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:54 smithi012 ceph-mon[37567]: osdmap e71: 8 total, 8 up, 8 in 2024-04-26T04:55:54.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:54 smithi012 ceph-mon[26617]: osdmap e71: 8 total, 8 up, 8 in 2024-04-26T04:55:54.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:54 smithi116 ceph-mon[29897]: osdmap e71: 8 total, 8 up, 8 in 2024-04-26T04:55:55.141 INFO:teuthology.orchestra.run.smithi012.stdout:55834574910 2024-04-26T04:55:55.517 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:55:55.694 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:55 smithi012 ceph-mon[26617]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:55.694 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:55 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/169768381' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T04:55:55.695 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:55 smithi012 ceph-mon[37567]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:55.695 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:55 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/169768381' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T04:55:55.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:55 smithi116 ceph-mon[29897]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:55.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:55 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/169768381' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T04:55:57.033 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986220 got 133143986222 for osd.4 2024-04-26T04:55:57.033 DEBUG:teuthology.parallel:result is None 2024-04-26T04:55:57.760 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:57 smithi012 ceph-mon[26617]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:57.760 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:57 smithi012 ceph-mon[37567]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:57.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:57 smithi116 ceph-mon[29897]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:58.903 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574906 got 55834574910 for osd.1 2024-04-26T04:55:58.903 DEBUG:teuthology.parallel:result is None 2024-04-26T04:55:59.654 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:55:59 smithi012 ceph-mon[26617]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:59.654 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:55:59 smithi012 ceph-mon[37567]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:55:59.794 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:55:59 smithi116 ceph-mon[29897]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:00.084 INFO:teuthology.orchestra.run.smithi012.stdout:167503724588 2024-04-26T04:56:00.276 INFO:teuthology.orchestra.run.smithi012.stdout:236223201314 2024-04-26T04:56:00.614 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:00 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/2171144684' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T04:56:00.614 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:00 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/2052111590' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T04:56:00.614 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:00 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/2171144684' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T04:56:00.614 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:00 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/2052111590' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T04:56:00.972 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:00 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/2171144684' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T04:56:00.972 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:00 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/2052111590' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T04:56:01.556 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724584 got 167503724588 for osd.5 2024-04-26T04:56:01.556 DEBUG:teuthology.parallel:result is None 2024-04-26T04:56:01.685 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:01 smithi012 ceph-mon[26617]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:01.685 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:01 smithi012 ceph-mon[37567]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:01.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:01 smithi116 ceph-mon[29897]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:02.421 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201311 got 236223201314 for osd.7 2024-04-26T04:56:02.421 DEBUG:teuthology.parallel:result is None 2024-04-26T04:56:03.004 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:56:02 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:56:02] "GET /metrics HTTP/1.1" 200 239045 "" "Prometheus/2.43.0" 2024-04-26T04:56:03.116 INFO:teuthology.orchestra.run.smithi012.stdout:201863462951 2024-04-26T04:56:03.316 INFO:teuthology.orchestra.run.smithi012.stdout:111669149748 2024-04-26T04:56:03.599 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:03 smithi012 ceph-mon[26617]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:03.599 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:03 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/2465707732' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T04:56:03.599 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:03 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1425065765' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T04:56:03.599 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:03 smithi012 ceph-mon[37567]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:03.599 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:03 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/2465707732' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T04:56:03.600 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:03 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/1425065765' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T04:56:03.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:03 smithi116 ceph-mon[29897]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:03.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:03 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/2465707732' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T04:56:03.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:03 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/1425065765' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T04:56:03.992 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:56:03 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:56:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:56:04.027 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462948 got 201863462951 for osd.6 2024-04-26T04:56:04.027 DEBUG:teuthology.parallel:result is None 2024-04-26T04:56:04.503 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149745 got 111669149748 for osd.3 2024-04-26T04:56:04.503 DEBUG:teuthology.parallel:result is None 2024-04-26T04:56:04.504 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-26T04:56:04.504 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-26T04:56:04.771 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:05.802 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:05 smithi012 ceph-mon[26617]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:05.803 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:05 smithi012 ceph-mon[37567]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:05.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:05 smithi116 ceph-mon[29897]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:06.527 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:56:06.528 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2024-04-26T04:56:07.300 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":81,"stamp":"2024-04-26T04:56:06.431423+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386348,"kb_used_data":7480,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747574868,"statfs":{"total":767960285184,"available":765516664832,"internally_reserved":0,"allocated":7659520,"data_stored":4308421,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002742"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.497927+0000","last_change":"2024-04-26T04:55:07.502061+0000","last_active":"2024-04-26T04:55:52.497927+0000","last_peered":"2024-04-26T04:55:52.497927+0000","last_clean":"2024-04-26T04:55:52.497927+0000","last_became_active":"2024-04-26T04:55:07.501751+0000","last_became_peered":"2024-04-26T04:55:07.501751+0000","last_unstale":"2024-04-26T04:55:52.497927+0000","last_undegraded":"2024-04-26T04:55:52.497927+0000","last_fullsized":"2024-04-26T04:55:52.497927+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:18:00.822676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.820585+0000","last_change":"2024-04-26T04:55:07.456603+0000","last_active":"2024-04-26T04:55:52.820585+0000","last_peered":"2024-04-26T04:55:52.820585+0000","last_clean":"2024-04-26T04:55:52.820585+0000","last_became_active":"2024-04-26T04:55:07.456287+0000","last_became_peered":"2024-04-26T04:55:07.456287+0000","last_unstale":"2024-04-26T04:55:52.820585+0000","last_undegraded":"2024-04-26T04:55:52.820585+0000","last_fullsized":"2024-04-26T04:55:52.820585+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:48:36.315805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498785+0000","last_change":"2024-04-26T04:55:08.476806+0000","last_active":"2024-04-26T04:55:52.498785+0000","last_peered":"2024-04-26T04:55:52.498785+0000","last_clean":"2024-04-26T04:55:52.498785+0000","last_became_active":"2024-04-26T04:55:08.476506+0000","last_became_peered":"2024-04-26T04:55:08.476506+0000","last_unstale":"2024-04-26T04:55:52.498785+0000","last_undegraded":"2024-04-26T04:55:52.498785+0000","last_fullsized":"2024-04-26T04:55:52.498785+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:55:21.535316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.499927+0000","last_change":"2024-04-26T04:55:07.456419+0000","last_active":"2024-04-26T04:55:52.499927+0000","last_peered":"2024-04-26T04:55:52.499927+0000","last_clean":"2024-04-26T04:55:52.499927+0000","last_became_active":"2024-04-26T04:55:07.455639+0000","last_became_peered":"2024-04-26T04:55:07.455639+0000","last_unstale":"2024-04-26T04:55:52.499927+0000","last_undegraded":"2024-04-26T04:55:52.499927+0000","last_fullsized":"2024-04-26T04:55:52.499927+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:11:53.038126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498508+0000","last_change":"2024-04-26T04:55:07.482572+0000","last_active":"2024-04-26T04:55:52.498508+0000","last_peered":"2024-04-26T04:55:52.498508+0000","last_clean":"2024-04-26T04:55:52.498508+0000","last_became_active":"2024-04-26T04:55:07.482371+0000","last_became_peered":"2024-04-26T04:55:07.482371+0000","last_unstale":"2024-04-26T04:55:52.498508+0000","last_undegraded":"2024-04-26T04:55:52.498508+0000","last_fullsized":"2024-04-26T04:55:52.498508+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:00:03.465526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.496543+0000","last_change":"2024-04-26T04:55:07.445793+0000","last_active":"2024-04-26T04:55:52.496543+0000","last_peered":"2024-04-26T04:55:52.496543+0000","last_clean":"2024-04-26T04:55:52.496543+0000","last_became_active":"2024-04-26T04:55:07.445496+0000","last_became_peered":"2024-04-26T04:55:07.445496+0000","last_unstale":"2024-04-26T04:55:52.496543+0000","last_undegraded":"2024-04-26T04:55:52.496543+0000","last_fullsized":"2024-04-26T04:55:52.496543+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:53:56.081693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-26T04:55:53.687991+0000","last_change":"2024-04-26T04:55:53.687991+0000","last_active":"2024-04-26T04:55:53.687991+0000","last_peered":"2024-04-26T04:55:53.687991+0000","last_clean":"2024-04-26T04:55:53.687991+0000","last_became_active":"2024-04-26T04:55:53.687560+0000","last_became_peered":"2024-04-26T04:55:53.687560+0000","last_unstale":"2024-04-26T04:55:53.687991+0000","last_undegraded":"2024-04-26T04:55:53.687991+0000","last_fullsized":"2024-04-26T04:55:53.687991+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:42:42.995605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.496269+0000","last_change":"2024-04-26T04:55:07.449303+0000","last_active":"2024-04-26T04:55:52.496269+0000","last_peered":"2024-04-26T04:55:52.496269+0000","last_clean":"2024-04-26T04:55:52.496269+0000","last_became_active":"2024-04-26T04:55:07.449039+0000","last_became_peered":"2024-04-26T04:55:07.449039+0000","last_unstale":"2024-04-26T04:55:52.496269+0000","last_undegraded":"2024-04-26T04:55:52.496269+0000","last_fullsized":"2024-04-26T04:55:52.496269+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:49:20.436815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.499259+0000","last_change":"2024-04-26T04:55:07.501687+0000","last_active":"2024-04-26T04:55:52.499259+0000","last_peered":"2024-04-26T04:55:52.499259+0000","last_clean":"2024-04-26T04:55:52.499259+0000","last_became_active":"2024-04-26T04:55:07.501318+0000","last_became_peered":"2024-04-26T04:55:07.501318+0000","last_unstale":"2024-04-26T04:55:52.499259+0000","last_undegraded":"2024-04-26T04:55:52.499259+0000","last_fullsized":"2024-04-26T04:55:52.499259+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:41:42.194849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.820456+0000","last_change":"2024-04-26T04:55:07.461008+0000","last_active":"2024-04-26T04:55:52.820456+0000","last_peered":"2024-04-26T04:55:52.820456+0000","last_clean":"2024-04-26T04:55:52.820456+0000","last_became_active":"2024-04-26T04:55:07.458834+0000","last_became_peered":"2024-04-26T04:55:07.458834+0000","last_unstale":"2024-04-26T04:55:52.820456+0000","last_undegraded":"2024-04-26T04:55:52.820456+0000","last_fullsized":"2024-04-26T04:55:52.820456+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:30:44.154385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498584+0000","last_change":"2024-04-26T04:55:07.474089+0000","last_active":"2024-04-26T04:55:52.498584+0000","last_peered":"2024-04-26T04:55:52.498584+0000","last_clean":"2024-04-26T04:55:52.498584+0000","last_became_active":"2024-04-26T04:55:07.473624+0000","last_became_peered":"2024-04-26T04:55:07.473624+0000","last_unstale":"2024-04-26T04:55:52.498584+0000","last_undegraded":"2024-04-26T04:55:52.498584+0000","last_fullsized":"2024-04-26T04:55:52.498584+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:15:47.507462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.820479+0000","last_change":"2024-04-26T04:55:07.458133+0000","last_active":"2024-04-26T04:55:52.820479+0000","last_peered":"2024-04-26T04:55:52.820479+0000","last_clean":"2024-04-26T04:55:52.820479+0000","last_became_active":"2024-04-26T04:55:07.457777+0000","last_became_peered":"2024-04-26T04:55:07.457777+0000","last_unstale":"2024-04-26T04:55:52.820479+0000","last_undegraded":"2024-04-26T04:55:52.820479+0000","last_fullsized":"2024-04-26T04:55:52.820479+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:21:53.116783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498998+0000","last_change":"2024-04-26T04:53:57.857083+0000","last_active":"2024-04-26T04:55:52.498998+0000","last_peered":"2024-04-26T04:55:52.498998+0000","last_clean":"2024-04-26T04:55:52.498998+0000","last_became_active":"2024-04-26T04:53:57.856775+0000","last_became_peered":"2024-04-26T04:53:57.856775+0000","last_unstale":"2024-04-26T04:55:52.498998+0000","last_undegraded":"2024-04-26T04:55:52.498998+0000","last_fullsized":"2024-04-26T04:55:52.498998+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:06:53.087376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498696+0000","last_change":"2024-04-26T04:55:08.601953+0000","last_active":"2024-04-26T04:55:52.498696+0000","last_peered":"2024-04-26T04:55:52.498696+0000","last_clean":"2024-04-26T04:55:52.498696+0000","last_became_active":"2024-04-26T04:55:08.601525+0000","last_became_peered":"2024-04-26T04:55:08.601525+0000","last_unstale":"2024-04-26T04:55:52.498696+0000","last_undegraded":"2024-04-26T04:55:52.498696+0000","last_fullsized":"2024-04-26T04:55:52.498696+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:28:47.387070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.496057+0000","last_change":"2024-04-26T04:53:57.938166+0000","last_active":"2024-04-26T04:55:52.496057+0000","last_peered":"2024-04-26T04:55:52.496057+0000","last_clean":"2024-04-26T04:55:52.496057+0000","last_became_active":"2024-04-26T04:53:57.937842+0000","last_became_peered":"2024-04-26T04:53:57.937842+0000","last_unstale":"2024-04-26T04:55:52.496057+0000","last_undegraded":"2024-04-26T04:55:52.496057+0000","last_fullsized":"2024-04-26T04:55:52.496057+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:39:45.320158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.500094+0000","last_change":"2024-04-26T04:55:07.464082+0000","last_active":"2024-04-26T04:55:52.500094+0000","last_peered":"2024-04-26T04:55:52.500094+0000","last_clean":"2024-04-26T04:55:52.500094+0000","last_became_active":"2024-04-26T04:55:07.463779+0000","last_became_peered":"2024-04-26T04:55:07.463779+0000","last_unstale":"2024-04-26T04:55:52.500094+0000","last_undegraded":"2024-04-26T04:55:52.500094+0000","last_fullsized":"2024-04-26T04:55:52.500094+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:08:24.246321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.820500+0000","last_change":"2024-04-26T04:55:08.603346+0000","last_active":"2024-04-26T04:55:52.820500+0000","last_peered":"2024-04-26T04:55:52.820500+0000","last_clean":"2024-04-26T04:55:52.820500+0000","last_became_active":"2024-04-26T04:55:08.603046+0000","last_became_peered":"2024-04-26T04:55:08.603046+0000","last_unstale":"2024-04-26T04:55:52.820500+0000","last_undegraded":"2024-04-26T04:55:52.820500+0000","last_fullsized":"2024-04-26T04:55:52.820500+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:50:37.364418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.495774+0000","last_change":"2024-04-26T04:55:07.450160+0000","last_active":"2024-04-26T04:55:52.495774+0000","last_peered":"2024-04-26T04:55:52.495774+0000","last_clean":"2024-04-26T04:55:52.495774+0000","last_became_active":"2024-04-26T04:55:07.449749+0000","last_became_peered":"2024-04-26T04:55:07.449749+0000","last_unstale":"2024-04-26T04:55:52.495774+0000","last_undegraded":"2024-04-26T04:55:52.495774+0000","last_fullsized":"2024-04-26T04:55:52.495774+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:22:48.334435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.501748+0000","last_change":"2024-04-26T04:53:57.939093+0000","last_active":"2024-04-26T04:55:52.501748+0000","last_peered":"2024-04-26T04:55:52.501748+0000","last_clean":"2024-04-26T04:55:52.501748+0000","last_became_active":"2024-04-26T04:53:57.938662+0000","last_became_peered":"2024-04-26T04:53:57.938662+0000","last_unstale":"2024-04-26T04:55:52.501748+0000","last_undegraded":"2024-04-26T04:55:52.501748+0000","last_fullsized":"2024-04-26T04:55:52.501748+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:37:22.972048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.820510+0000","last_change":"2024-04-26T04:55:08.476125+0000","last_active":"2024-04-26T04:55:52.820510+0000","last_peered":"2024-04-26T04:55:52.820510+0000","last_clean":"2024-04-26T04:55:52.820510+0000","last_became_active":"2024-04-26T04:55:08.475766+0000","last_became_peered":"2024-04-26T04:55:08.475766+0000","last_unstale":"2024-04-26T04:55:52.820510+0000","last_undegraded":"2024-04-26T04:55:52.820510+0000","last_fullsized":"2024-04-26T04:55:52.820510+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:46:44.083324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.820305+0000","last_change":"2024-04-26T04:55:07.453608+0000","last_active":"2024-04-26T04:55:52.820305+0000","last_peered":"2024-04-26T04:55:52.820305+0000","last_clean":"2024-04-26T04:55:52.820305+0000","last_became_active":"2024-04-26T04:55:07.453310+0000","last_became_peered":"2024-04-26T04:55:07.453310+0000","last_unstale":"2024-04-26T04:55:52.820305+0000","last_undegraded":"2024-04-26T04:55:52.820305+0000","last_fullsized":"2024-04-26T04:55:52.820305+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:33:40.865186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.501473+0000","last_change":"2024-04-26T04:53:57.939087+0000","last_active":"2024-04-26T04:55:52.501473+0000","last_peered":"2024-04-26T04:55:52.501473+0000","last_clean":"2024-04-26T04:55:52.501473+0000","last_became_active":"2024-04-26T04:53:57.938647+0000","last_became_peered":"2024-04-26T04:53:57.938647+0000","last_unstale":"2024-04-26T04:55:52.501473+0000","last_undegraded":"2024-04-26T04:55:52.501473+0000","last_fullsized":"2024-04-26T04:55:52.501473+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:51:38.687896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.499065+0000","last_change":"2024-04-26T04:55:07.500866+0000","last_active":"2024-04-26T04:55:52.499065+0000","last_peered":"2024-04-26T04:55:52.499065+0000","last_clean":"2024-04-26T04:55:52.499065+0000","last_became_active":"2024-04-26T04:55:07.500516+0000","last_became_peered":"2024-04-26T04:55:07.500516+0000","last_unstale":"2024-04-26T04:55:52.499065+0000","last_undegraded":"2024-04-26T04:55:52.499065+0000","last_fullsized":"2024-04-26T04:55:52.499065+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:10:30.230397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.499274+0000","last_change":"2024-04-26T04:55:07.456313+0000","last_active":"2024-04-26T04:55:52.499274+0000","last_peered":"2024-04-26T04:55:52.499274+0000","last_clean":"2024-04-26T04:55:52.499274+0000","last_became_active":"2024-04-26T04:55:07.456037+0000","last_became_peered":"2024-04-26T04:55:07.456037+0000","last_unstale":"2024-04-26T04:55:52.499274+0000","last_undegraded":"2024-04-26T04:55:52.499274+0000","last_fullsized":"2024-04-26T04:55:52.499274+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:57:09.971101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":278,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.501397+0000","last_change":"2024-04-26T04:53:25.550145+0000","last_active":"2024-04-26T04:55:52.501397+0000","last_peered":"2024-04-26T04:55:52.501397+0000","last_clean":"2024-04-26T04:55:52.501397+0000","last_became_active":"2024-04-26T04:53:25.549600+0000","last_became_peered":"2024-04-26T04:53:25.549600+0000","last_unstale":"2024-04-26T04:55:52.501397+0000","last_undegraded":"2024-04-26T04:55:52.501397+0000","last_fullsized":"2024-04-26T04:55:52.501397+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:51:27.309540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:51:27.309540+0000","last_clean_scrub_stamp":"2024-04-26T04:51:27.309540+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:23:44.865329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.496364+0000","last_change":"2024-04-26T04:53:57.856241+0000","last_active":"2024-04-26T04:55:52.496364+0000","last_peered":"2024-04-26T04:55:52.496364+0000","last_clean":"2024-04-26T04:55:52.496364+0000","last_became_active":"2024-04-26T04:53:57.855745+0000","last_became_peered":"2024-04-26T04:53:57.855745+0000","last_unstale":"2024-04-26T04:55:52.496364+0000","last_undegraded":"2024-04-26T04:55:52.496364+0000","last_fullsized":"2024-04-26T04:55:52.496364+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:54:03.208950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.496217+0000","last_change":"2024-04-26T04:55:08.478015+0000","last_active":"2024-04-26T04:55:52.496217+0000","last_peered":"2024-04-26T04:55:52.496217+0000","last_clean":"2024-04-26T04:55:52.496217+0000","last_became_active":"2024-04-26T04:55:08.477711+0000","last_became_peered":"2024-04-26T04:55:08.477711+0000","last_unstale":"2024-04-26T04:55:52.496217+0000","last_undegraded":"2024-04-26T04:55:52.496217+0000","last_fullsized":"2024-04-26T04:55:52.496217+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T04:57:44.192610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.495349+0000","last_change":"2024-04-26T04:55:07.452343+0000","last_active":"2024-04-26T04:55:52.495349+0000","last_peered":"2024-04-26T04:55:52.495349+0000","last_clean":"2024-04-26T04:55:52.495349+0000","last_became_active":"2024-04-26T04:55:07.452057+0000","last_became_peered":"2024-04-26T04:55:07.452057+0000","last_unstale":"2024-04-26T04:55:52.495349+0000","last_undegraded":"2024-04-26T04:55:52.495349+0000","last_fullsized":"2024-04-26T04:55:52.495349+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:01:49.065366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498813+0000","last_change":"2024-04-26T04:53:57.939187+0000","last_active":"2024-04-26T04:55:52.498813+0000","last_peered":"2024-04-26T04:55:52.498813+0000","last_clean":"2024-04-26T04:55:52.498813+0000","last_became_active":"2024-04-26T04:53:57.938749+0000","last_became_peered":"2024-04-26T04:53:57.938749+0000","last_unstale":"2024-04-26T04:55:52.498813+0000","last_undegraded":"2024-04-26T04:55:52.498813+0000","last_fullsized":"2024-04-26T04:55:52.498813+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:59:54.974432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.495694+0000","last_change":"2024-04-26T04:55:08.474447+0000","last_active":"2024-04-26T04:55:52.495694+0000","last_peered":"2024-04-26T04:55:52.495694+0000","last_clean":"2024-04-26T04:55:52.495694+0000","last_became_active":"2024-04-26T04:55:08.474159+0000","last_became_peered":"2024-04-26T04:55:08.474159+0000","last_unstale":"2024-04-26T04:55:52.495694+0000","last_undegraded":"2024-04-26T04:55:52.495694+0000","last_fullsized":"2024-04-26T04:55:52.495694+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:18:45.625120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498169+0000","last_change":"2024-04-26T04:55:07.457209+0000","last_active":"2024-04-26T04:55:52.498169+0000","last_peered":"2024-04-26T04:55:52.498169+0000","last_clean":"2024-04-26T04:55:52.498169+0000","last_became_active":"2024-04-26T04:55:07.456997+0000","last_became_peered":"2024-04-26T04:55:07.456997+0000","last_unstale":"2024-04-26T04:55:52.498169+0000","last_undegraded":"2024-04-26T04:55:52.498169+0000","last_fullsized":"2024-04-26T04:55:52.498169+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:27:28.288028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.496164+0000","last_change":"2024-04-26T04:53:57.856053+0000","last_active":"2024-04-26T04:55:52.496164+0000","last_peered":"2024-04-26T04:55:52.496164+0000","last_clean":"2024-04-26T04:55:52.496164+0000","last_became_active":"2024-04-26T04:53:57.855693+0000","last_became_peered":"2024-04-26T04:53:57.855693+0000","last_unstale":"2024-04-26T04:55:52.496164+0000","last_undegraded":"2024-04-26T04:55:52.496164+0000","last_fullsized":"2024-04-26T04:55:52.496164+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:35:20.404397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.495422+0000","last_change":"2024-04-26T04:55:08.569282+0000","last_active":"2024-04-26T04:55:52.495422+0000","last_peered":"2024-04-26T04:55:52.495422+0000","last_clean":"2024-04-26T04:55:52.495422+0000","last_became_active":"2024-04-26T04:55:08.568536+0000","last_became_peered":"2024-04-26T04:55:08.568536+0000","last_unstale":"2024-04-26T04:55:52.495422+0000","last_undegraded":"2024-04-26T04:55:52.495422+0000","last_fullsized":"2024-04-26T04:55:52.495422+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:55:18.326130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.496511+0000","last_change":"2024-04-26T04:55:07.448584+0000","last_active":"2024-04-26T04:55:52.496511+0000","last_peered":"2024-04-26T04:55:52.496511+0000","last_clean":"2024-04-26T04:55:52.496511+0000","last_became_active":"2024-04-26T04:55:07.447767+0000","last_became_peered":"2024-04-26T04:55:07.447767+0000","last_unstale":"2024-04-26T04:55:52.496511+0000","last_undegraded":"2024-04-26T04:55:52.496511+0000","last_fullsized":"2024-04-26T04:55:52.496511+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:51:17.365374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.495618+0000","last_change":"2024-04-26T04:53:57.939812+0000","last_active":"2024-04-26T04:55:52.495618+0000","last_peered":"2024-04-26T04:55:52.495618+0000","last_clean":"2024-04-26T04:55:52.495618+0000","last_became_active":"2024-04-26T04:53:57.939509+0000","last_became_peered":"2024-04-26T04:53:57.939509+0000","last_unstale":"2024-04-26T04:55:52.495618+0000","last_undegraded":"2024-04-26T04:55:52.495618+0000","last_fullsized":"2024-04-26T04:55:52.495618+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:05:41.872666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.496143+0000","last_change":"2024-04-26T04:55:08.568905+0000","last_active":"2024-04-26T04:55:52.496143+0000","last_peered":"2024-04-26T04:55:52.496143+0000","last_clean":"2024-04-26T04:55:52.496143+0000","last_became_active":"2024-04-26T04:55:08.568603+0000","last_became_peered":"2024-04-26T04:55:08.568603+0000","last_unstale":"2024-04-26T04:55:52.496143+0000","last_undegraded":"2024-04-26T04:55:52.496143+0000","last_fullsized":"2024-04-26T04:55:52.496143+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:14:11.351749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498784+0000","last_change":"2024-04-26T04:55:07.457164+0000","last_active":"2024-04-26T04:55:52.498784+0000","last_peered":"2024-04-26T04:55:52.498784+0000","last_clean":"2024-04-26T04:55:52.498784+0000","last_became_active":"2024-04-26T04:55:07.456826+0000","last_became_peered":"2024-04-26T04:55:07.456826+0000","last_unstale":"2024-04-26T04:55:52.498784+0000","last_undegraded":"2024-04-26T04:55:52.498784+0000","last_fullsized":"2024-04-26T04:55:52.498784+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:00:42.598533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498791+0000","last_change":"2024-04-26T04:55:07.502223+0000","last_active":"2024-04-26T04:55:52.498791+0000","last_peered":"2024-04-26T04:55:52.498791+0000","last_clean":"2024-04-26T04:55:52.498791+0000","last_became_active":"2024-04-26T04:55:07.501930+0000","last_became_peered":"2024-04-26T04:55:07.501930+0000","last_unstale":"2024-04-26T04:55:52.498791+0000","last_undegraded":"2024-04-26T04:55:52.498791+0000","last_fullsized":"2024-04-26T04:55:52.498791+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:01:41.107020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498392+0000","last_change":"2024-04-26T04:55:07.451911+0000","last_active":"2024-04-26T04:55:52.498392+0000","last_peered":"2024-04-26T04:55:52.498392+0000","last_clean":"2024-04-26T04:55:52.498392+0000","last_became_active":"2024-04-26T04:55:07.451612+0000","last_became_peered":"2024-04-26T04:55:07.451612+0000","last_unstale":"2024-04-26T04:55:52.498392+0000","last_undegraded":"2024-04-26T04:55:52.498392+0000","last_fullsized":"2024-04-26T04:55:52.498392+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:38:48.221948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.499125+0000","last_change":"2024-04-26T04:55:07.502561+0000","last_active":"2024-04-26T04:55:52.499125+0000","last_peered":"2024-04-26T04:55:52.499125+0000","last_clean":"2024-04-26T04:55:52.499125+0000","last_became_active":"2024-04-26T04:55:07.501680+0000","last_became_peered":"2024-04-26T04:55:07.501680+0000","last_unstale":"2024-04-26T04:55:52.499125+0000","last_undegraded":"2024-04-26T04:55:52.499125+0000","last_fullsized":"2024-04-26T04:55:52.499125+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:56:56.062544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.499947+0000","last_change":"2024-04-26T04:55:07.450789+0000","last_active":"2024-04-26T04:55:52.499947+0000","last_peered":"2024-04-26T04:55:52.499947+0000","last_clean":"2024-04-26T04:55:52.499947+0000","last_became_active":"2024-04-26T04:55:07.450560+0000","last_became_peered":"2024-04-26T04:55:07.450560+0000","last_unstale":"2024-04-26T04:55:52.499947+0000","last_undegraded":"2024-04-26T04:55:52.499947+0000","last_fullsized":"2024-04-26T04:55:52.499947+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:10:49.656045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498227+0000","last_change":"2024-04-26T04:55:07.474296+0000","last_active":"2024-04-26T04:55:52.498227+0000","last_peered":"2024-04-26T04:55:52.498227+0000","last_clean":"2024-04-26T04:55:52.498227+0000","last_became_active":"2024-04-26T04:55:07.473638+0000","last_became_peered":"2024-04-26T04:55:07.473638+0000","last_unstale":"2024-04-26T04:55:52.498227+0000","last_undegraded":"2024-04-26T04:55:52.498227+0000","last_fullsized":"2024-04-26T04:55:52.498227+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:57:58.375045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.499334+0000","last_change":"2024-04-26T04:55:07.458488+0000","last_active":"2024-04-26T04:55:52.499334+0000","last_peered":"2024-04-26T04:55:52.499334+0000","last_clean":"2024-04-26T04:55:52.499334+0000","last_became_active":"2024-04-26T04:55:07.458201+0000","last_became_peered":"2024-04-26T04:55:07.458201+0000","last_unstale":"2024-04-26T04:55:52.499334+0000","last_undegraded":"2024-04-26T04:55:52.499334+0000","last_fullsized":"2024-04-26T04:55:52.499334+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:43:41.715413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.820403+0000","last_change":"2024-04-26T04:55:08.568873+0000","last_active":"2024-04-26T04:55:52.820403+0000","last_peered":"2024-04-26T04:55:52.820403+0000","last_clean":"2024-04-26T04:55:52.820403+0000","last_became_active":"2024-04-26T04:55:08.568566+0000","last_became_peered":"2024-04-26T04:55:08.568566+0000","last_unstale":"2024-04-26T04:55:52.820403+0000","last_undegraded":"2024-04-26T04:55:52.820403+0000","last_fullsized":"2024-04-26T04:55:52.820403+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:51:27.194663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.496231+0000","last_change":"2024-04-26T04:55:07.448595+0000","last_active":"2024-04-26T04:55:52.496231+0000","last_peered":"2024-04-26T04:55:52.496231+0000","last_clean":"2024-04-26T04:55:52.496231+0000","last_became_active":"2024-04-26T04:55:07.448321+0000","last_became_peered":"2024-04-26T04:55:07.448321+0000","last_unstale":"2024-04-26T04:55:52.496231+0000","last_undegraded":"2024-04-26T04:55:52.496231+0000","last_fullsized":"2024-04-26T04:55:52.496231+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:10:46.543937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.499829+0000","last_change":"2024-04-26T04:55:07.502274+0000","last_active":"2024-04-26T04:55:52.499829+0000","last_peered":"2024-04-26T04:55:52.499829+0000","last_clean":"2024-04-26T04:55:52.499829+0000","last_became_active":"2024-04-26T04:55:07.501989+0000","last_became_peered":"2024-04-26T04:55:07.501989+0000","last_unstale":"2024-04-26T04:55:52.499829+0000","last_undegraded":"2024-04-26T04:55:52.499829+0000","last_fullsized":"2024-04-26T04:55:52.499829+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:07:10.637739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.501793+0000","last_change":"2024-04-26T04:55:07.451952+0000","last_active":"2024-04-26T04:55:52.501793+0000","last_peered":"2024-04-26T04:55:52.501793+0000","last_clean":"2024-04-26T04:55:52.501793+0000","last_became_active":"2024-04-26T04:55:07.451660+0000","last_became_peered":"2024-04-26T04:55:07.451660+0000","last_unstale":"2024-04-26T04:55:52.501793+0000","last_undegraded":"2024-04-26T04:55:52.501793+0000","last_fullsized":"2024-04-26T04:55:52.501793+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:36:14.683401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498146+0000","last_change":"2024-04-26T04:55:07.501239+0000","last_active":"2024-04-26T04:55:52.498146+0000","last_peered":"2024-04-26T04:55:52.498146+0000","last_clean":"2024-04-26T04:55:52.498146+0000","last_became_active":"2024-04-26T04:55:07.500968+0000","last_became_peered":"2024-04-26T04:55:07.500968+0000","last_unstale":"2024-04-26T04:55:52.498146+0000","last_undegraded":"2024-04-26T04:55:52.498146+0000","last_fullsized":"2024-04-26T04:55:52.498146+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:39:01.835409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.501758+0000","last_change":"2024-04-26T04:55:07.453480+0000","last_active":"2024-04-26T04:55:52.501758+0000","last_peered":"2024-04-26T04:55:52.501758+0000","last_clean":"2024-04-26T04:55:52.501758+0000","last_became_active":"2024-04-26T04:55:07.453141+0000","last_became_peered":"2024-04-26T04:55:07.453141+0000","last_unstale":"2024-04-26T04:55:52.501758+0000","last_undegraded":"2024-04-26T04:55:52.501758+0000","last_fullsized":"2024-04-26T04:55:52.501758+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:33:34.384745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.496079+0000","last_change":"2024-04-26T04:55:07.500711+0000","last_active":"2024-04-26T04:55:52.496079+0000","last_peered":"2024-04-26T04:55:52.496079+0000","last_clean":"2024-04-26T04:55:52.496079+0000","last_became_active":"2024-04-26T04:55:07.500428+0000","last_became_peered":"2024-04-26T04:55:07.500428+0000","last_unstale":"2024-04-26T04:55:52.496079+0000","last_undegraded":"2024-04-26T04:55:52.496079+0000","last_fullsized":"2024-04-26T04:55:52.496079+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:07:04.979529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.502029+0000","last_change":"2024-04-26T04:55:07.463656+0000","last_active":"2024-04-26T04:55:52.502029+0000","last_peered":"2024-04-26T04:55:52.502029+0000","last_clean":"2024-04-26T04:55:52.502029+0000","last_became_active":"2024-04-26T04:55:07.463417+0000","last_became_peered":"2024-04-26T04:55:07.463417+0000","last_unstale":"2024-04-26T04:55:52.502029+0000","last_undegraded":"2024-04-26T04:55:52.502029+0000","last_fullsized":"2024-04-26T04:55:52.502029+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:56:02.866229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-26T04:55:53.687811+0000","last_change":"2024-04-26T04:55:53.687811+0000","last_active":"2024-04-26T04:55:53.687811+0000","last_peered":"2024-04-26T04:55:53.687811+0000","last_clean":"2024-04-26T04:55:53.687811+0000","last_became_active":"2024-04-26T04:55:53.687402+0000","last_became_peered":"2024-04-26T04:55:53.687402+0000","last_unstale":"2024-04-26T04:55:53.687811+0000","last_undegraded":"2024-04-26T04:55:53.687811+0000","last_fullsized":"2024-04-26T04:55:53.687811+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:04:02.155272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":22,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498864+0000","last_change":"2024-04-26T04:55:07.455662+0000","last_active":"2024-04-26T04:55:52.498864+0000","last_peered":"2024-04-26T04:55:52.498864+0000","last_clean":"2024-04-26T04:55:52.498864+0000","last_became_active":"2024-04-26T04:55:07.455401+0000","last_became_peered":"2024-04-26T04:55:07.455401+0000","last_unstale":"2024-04-26T04:55:52.498864+0000","last_undegraded":"2024-04-26T04:55:52.498864+0000","last_fullsized":"2024-04-26T04:55:52.498864+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:41:47.842841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.820372+0000","last_change":"2024-04-26T04:55:08.602361+0000","last_active":"2024-04-26T04:55:52.820372+0000","last_peered":"2024-04-26T04:55:52.820372+0000","last_clean":"2024-04-26T04:55:52.820372+0000","last_became_active":"2024-04-26T04:55:08.602094+0000","last_became_peered":"2024-04-26T04:55:08.602094+0000","last_unstale":"2024-04-26T04:55:52.820372+0000","last_undegraded":"2024-04-26T04:55:52.820372+0000","last_fullsized":"2024-04-26T04:55:52.820372+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:07:17.911220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.501367+0000","last_change":"2024-04-26T04:55:07.452729+0000","last_active":"2024-04-26T04:55:52.501367+0000","last_peered":"2024-04-26T04:55:52.501367+0000","last_clean":"2024-04-26T04:55:52.501367+0000","last_became_active":"2024-04-26T04:55:07.452426+0000","last_became_peered":"2024-04-26T04:55:07.452426+0000","last_unstale":"2024-04-26T04:55:52.501367+0000","last_undegraded":"2024-04-26T04:55:52.501367+0000","last_fullsized":"2024-04-26T04:55:52.501367+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:59:48.509688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.496372+0000","last_change":"2024-04-26T04:55:07.501192+0000","last_active":"2024-04-26T04:55:52.496372+0000","last_peered":"2024-04-26T04:55:52.496372+0000","last_clean":"2024-04-26T04:55:52.496372+0000","last_became_active":"2024-04-26T04:55:07.500547+0000","last_became_peered":"2024-04-26T04:55:07.500547+0000","last_unstale":"2024-04-26T04:55:52.496372+0000","last_undegraded":"2024-04-26T04:55:52.496372+0000","last_fullsized":"2024-04-26T04:55:52.496372+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:02:18.503080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.820415+0000","last_change":"2024-04-26T04:55:07.462667+0000","last_active":"2024-04-26T04:55:52.820415+0000","last_peered":"2024-04-26T04:55:52.820415+0000","last_clean":"2024-04-26T04:55:52.820415+0000","last_became_active":"2024-04-26T04:55:07.462424+0000","last_became_peered":"2024-04-26T04:55:07.462424+0000","last_unstale":"2024-04-26T04:55:52.820415+0000","last_undegraded":"2024-04-26T04:55:52.820415+0000","last_fullsized":"2024-04-26T04:55:52.820415+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:25:20.792932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.499455+0000","last_change":"2024-04-26T04:55:08.602806+0000","last_active":"2024-04-26T04:55:52.499455+0000","last_peered":"2024-04-26T04:55:52.499455+0000","last_clean":"2024-04-26T04:55:52.499455+0000","last_became_active":"2024-04-26T04:55:08.602515+0000","last_became_peered":"2024-04-26T04:55:08.602515+0000","last_unstale":"2024-04-26T04:55:52.499455+0000","last_undegraded":"2024-04-26T04:55:52.499455+0000","last_fullsized":"2024-04-26T04:55:52.499455+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:45:27.150544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.501675+0000","last_change":"2024-04-26T04:55:07.452078+0000","last_active":"2024-04-26T04:55:52.501675+0000","last_peered":"2024-04-26T04:55:52.501675+0000","last_clean":"2024-04-26T04:55:52.501675+0000","last_became_active":"2024-04-26T04:55:07.451798+0000","last_became_peered":"2024-04-26T04:55:07.451798+0000","last_unstale":"2024-04-26T04:55:52.501675+0000","last_undegraded":"2024-04-26T04:55:52.501675+0000","last_fullsized":"2024-04-26T04:55:52.501675+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:36:16.277372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498265+0000","last_change":"2024-04-26T04:55:07.503315+0000","last_active":"2024-04-26T04:55:52.498265+0000","last_peered":"2024-04-26T04:55:52.498265+0000","last_clean":"2024-04-26T04:55:52.498265+0000","last_became_active":"2024-04-26T04:55:07.502836+0000","last_became_peered":"2024-04-26T04:55:07.502836+0000","last_unstale":"2024-04-26T04:55:52.498265+0000","last_undegraded":"2024-04-26T04:55:52.498265+0000","last_fullsized":"2024-04-26T04:55:52.498265+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:28:24.192527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"69'6","reported_seq":20,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498407+0000","last_change":"2024-04-26T04:55:07.451574+0000","last_active":"2024-04-26T04:55:52.498407+0000","last_peered":"2024-04-26T04:55:52.498407+0000","last_clean":"2024-04-26T04:55:52.498407+0000","last_became_active":"2024-04-26T04:55:07.451154+0000","last_became_peered":"2024-04-26T04:55:07.451154+0000","last_unstale":"2024-04-26T04:55:52.498407+0000","last_undegraded":"2024-04-26T04:55:52.498407+0000","last_fullsized":"2024-04-26T04:55:52.498407+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:56:04.497071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":10,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-26T04:55:53.688486+0000","last_change":"2024-04-26T04:55:53.688486+0000","last_active":"2024-04-26T04:55:53.688486+0000","last_peered":"2024-04-26T04:55:53.688486+0000","last_clean":"2024-04-26T04:55:53.688486+0000","last_became_active":"2024-04-26T04:55:53.687991+0000","last_became_peered":"2024-04-26T04:55:53.687991+0000","last_unstale":"2024-04-26T04:55:53.688486+0000","last_undegraded":"2024-04-26T04:55:53.688486+0000","last_fullsized":"2024-04-26T04:55:53.688486+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:19:30.036450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.501685+0000","last_change":"2024-04-26T04:55:07.454011+0000","last_active":"2024-04-26T04:55:52.501685+0000","last_peered":"2024-04-26T04:55:52.501685+0000","last_clean":"2024-04-26T04:55:52.501685+0000","last_became_active":"2024-04-26T04:55:07.453695+0000","last_became_peered":"2024-04-26T04:55:07.453695+0000","last_unstale":"2024-04-26T04:55:52.501685+0000","last_undegraded":"2024-04-26T04:55:52.501685+0000","last_fullsized":"2024-04-26T04:55:52.501685+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:58:20.724910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.499072+0000","last_change":"2024-04-26T04:55:08.601188+0000","last_active":"2024-04-26T04:55:52.499072+0000","last_peered":"2024-04-26T04:55:52.499072+0000","last_clean":"2024-04-26T04:55:52.499072+0000","last_became_active":"2024-04-26T04:55:08.600844+0000","last_became_peered":"2024-04-26T04:55:08.600844+0000","last_unstale":"2024-04-26T04:55:52.499072+0000","last_undegraded":"2024-04-26T04:55:52.499072+0000","last_fullsized":"2024-04-26T04:55:52.499072+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:16:31.788265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.496409+0000","last_change":"2024-04-26T04:55:07.447684+0000","last_active":"2024-04-26T04:55:52.496409+0000","last_peered":"2024-04-26T04:55:52.496409+0000","last_clean":"2024-04-26T04:55:52.496409+0000","last_became_active":"2024-04-26T04:55:07.447380+0000","last_became_peered":"2024-04-26T04:55:07.447380+0000","last_unstale":"2024-04-26T04:55:52.496409+0000","last_undegraded":"2024-04-26T04:55:52.496409+0000","last_fullsized":"2024-04-26T04:55:52.496409+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:21:18.903588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498856+0000","last_change":"2024-04-26T04:55:08.601318+0000","last_active":"2024-04-26T04:55:52.498856+0000","last_peered":"2024-04-26T04:55:52.498856+0000","last_clean":"2024-04-26T04:55:52.498856+0000","last_became_active":"2024-04-26T04:55:08.601034+0000","last_became_peered":"2024-04-26T04:55:08.601034+0000","last_unstale":"2024-04-26T04:55:52.498856+0000","last_undegraded":"2024-04-26T04:55:52.498856+0000","last_fullsized":"2024-04-26T04:55:52.498856+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:08:22.624996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.820585+0000","last_change":"2024-04-26T04:55:07.457337+0000","last_active":"2024-04-26T04:55:52.820585+0000","last_peered":"2024-04-26T04:55:52.820585+0000","last_clean":"2024-04-26T04:55:52.820585+0000","last_became_active":"2024-04-26T04:55:07.456996+0000","last_became_peered":"2024-04-26T04:55:07.456996+0000","last_unstale":"2024-04-26T04:55:52.820585+0000","last_undegraded":"2024-04-26T04:55:52.820585+0000","last_fullsized":"2024-04-26T04:55:52.820585+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:26:34.645847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498905+0000","last_change":"2024-04-26T04:55:07.503910+0000","last_active":"2024-04-26T04:55:52.498905+0000","last_peered":"2024-04-26T04:55:52.498905+0000","last_clean":"2024-04-26T04:55:52.498905+0000","last_became_active":"2024-04-26T04:55:07.503589+0000","last_became_peered":"2024-04-26T04:55:07.503589+0000","last_unstale":"2024-04-26T04:55:52.498905+0000","last_undegraded":"2024-04-26T04:55:52.498905+0000","last_fullsized":"2024-04-26T04:55:52.498905+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:44:38.391157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498945+0000","last_change":"2024-04-26T04:55:07.461775+0000","last_active":"2024-04-26T04:55:52.498945+0000","last_peered":"2024-04-26T04:55:52.498945+0000","last_clean":"2024-04-26T04:55:52.498945+0000","last_became_active":"2024-04-26T04:55:07.461493+0000","last_became_peered":"2024-04-26T04:55:07.461493+0000","last_unstale":"2024-04-26T04:55:52.498945+0000","last_undegraded":"2024-04-26T04:55:52.498945+0000","last_fullsized":"2024-04-26T04:55:52.498945+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:00:46.837734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.820396+0000","last_change":"2024-04-26T04:55:08.601565+0000","last_active":"2024-04-26T04:55:52.820396+0000","last_peered":"2024-04-26T04:55:52.820396+0000","last_clean":"2024-04-26T04:55:52.820396+0000","last_became_active":"2024-04-26T04:55:08.601226+0000","last_became_peered":"2024-04-26T04:55:08.601226+0000","last_unstale":"2024-04-26T04:55:52.820396+0000","last_undegraded":"2024-04-26T04:55:52.820396+0000","last_fullsized":"2024-04-26T04:55:52.820396+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:15:30.591873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.495323+0000","last_change":"2024-04-26T04:55:07.457162+0000","last_active":"2024-04-26T04:55:52.495323+0000","last_peered":"2024-04-26T04:55:52.495323+0000","last_clean":"2024-04-26T04:55:52.495323+0000","last_became_active":"2024-04-26T04:55:07.456801+0000","last_became_peered":"2024-04-26T04:55:07.456801+0000","last_unstale":"2024-04-26T04:55:52.495323+0000","last_undegraded":"2024-04-26T04:55:52.495323+0000","last_fullsized":"2024-04-26T04:55:52.495323+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:39:42.692710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.499622+0000","last_change":"2024-04-26T04:55:08.569011+0000","last_active":"2024-04-26T04:55:52.499622+0000","last_peered":"2024-04-26T04:55:52.499622+0000","last_clean":"2024-04-26T04:55:52.499622+0000","last_became_active":"2024-04-26T04:55:08.568533+0000","last_became_peered":"2024-04-26T04:55:08.568533+0000","last_unstale":"2024-04-26T04:55:52.499622+0000","last_undegraded":"2024-04-26T04:55:52.499622+0000","last_fullsized":"2024-04-26T04:55:52.499622+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:53:08.464353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498275+0000","last_change":"2024-04-26T04:55:07.451596+0000","last_active":"2024-04-26T04:55:52.498275+0000","last_peered":"2024-04-26T04:55:52.498275+0000","last_clean":"2024-04-26T04:55:52.498275+0000","last_became_active":"2024-04-26T04:55:07.451265+0000","last_became_peered":"2024-04-26T04:55:07.451265+0000","last_unstale":"2024-04-26T04:55:52.498275+0000","last_undegraded":"2024-04-26T04:55:52.498275+0000","last_fullsized":"2024-04-26T04:55:52.498275+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:26:45.706484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201315,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298672,"kb_used_data":1312,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446480,"statfs":{"total":95995035648,"available":95689195520,"internally_reserved":0,"allocated":1343488,"data_stored":908282,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:55:26 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.54200000000000004,"15min":0.54400000000000004},"min":{"1min":0.33000000000000002,"5min":0.315,"15min":0.315},"max":{"1min":0.67700000000000005,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.503,"5min":0.51900000000000002,"15min":0.52100000000000002},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.61299999999999999}]},{"osd":1,"last update":"Fri Apr 26 04:55:26 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53900000000000003,"15min":0.54000000000000004},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.57299999999999995}]},{"osd":2,"last update":"Fri Apr 26 04:55:26 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55900000000000005,"15min":0.56000000000000005},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.49099999999999999}]},{"osd":3,"last update":"Fri Apr 26 04:55:26 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.58599999999999997}]},{"osd":4,"last update":"Fri Apr 26 04:55:26 2024","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.45300000000000001,"15min":0.44900000000000001},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.495,"15min":0.49099999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.42399999999999999}]},{"osd":5,"last update":"Fri Apr 26 04:55:26 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.48499999999999999,"15min":0.47799999999999998},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.504},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.51300000000000001,"15min":0.50700000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.47899999999999998}]},{"osd":6,"last update":"Fri Apr 26 04:55:26 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.53000000000000003,"15min":0.52600000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.69899999999999995,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.57199999999999995,"15min":0.56799999999999995},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.76200000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.52300000000000002}]}]},{"osd":6,"up_from":47,"seq":201863462952,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298652,"kb_used_data":1296,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446500,"statfs":{"total":95995035648,"available":95689216000,"internally_reserved":0,"allocated":1327104,"data_stored":907921,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:56:00 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.53600000000000003,"15min":0.53400000000000003},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.755,"5min":0.57999999999999996,"15min":0.55000000000000004},"min":{"1min":0.39800000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":4.6219999999999999,"5min":4.6219999999999999,"15min":4.6219999999999999},"last":0.48099999999999998}]},{"osd":1,"last update":"Fri Apr 26 04:56:00 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.76400000000000001,"15min":0.60399999999999998},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.745,"5min":18.579000000000001,"15min":18.579000000000001},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.52300000000000002,"15min":0.51400000000000001},"min":{"1min":0.42899999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.46000000000000002}]},{"osd":2,"last update":"Fri Apr 26 04:56:00 2024","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":0.74299999999999999,"15min":0.77300000000000002},"min":{"1min":0.39800000000000002,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":4.5789999999999997,"5min":5.6980000000000004,"15min":5.6980000000000004},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.434,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.73299999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.45400000000000001}]},{"osd":3,"last update":"Fri Apr 26 04:56:00 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54500000000000004},"min":{"1min":0.42899999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.71699999999999997,"15min":0.77300000000000002},"min":{"1min":0.48099999999999998,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.72199999999999998,"5min":5.6369999999999996,"15min":5.6369999999999996},"last":0.56399999999999995}]},{"osd":4,"last update":"Fri Apr 26 04:56:00 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.442,"15min":0.42499999999999999},"min":{"1min":0.29399999999999998,"5min":0.255,"15min":0.255},"max":{"1min":0.66800000000000004,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.46100000000000002,"15min":0.44},"min":{"1min":0.311,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.35999999999999999}]},{"osd":5,"last update":"Fri Apr 26 04:56:00 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.45700000000000002,"15min":0.435},"min":{"1min":0.33200000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.65000000000000002,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.47599999999999998,"15min":0.45200000000000001},"min":{"1min":0.38500000000000001,"5min":0.308,"15min":0.308},"max":{"1min":0.73899999999999999,"5min":0.753,"15min":0.753},"last":0.40500000000000003}]},{"osd":7,"last update":"Fri Apr 26 04:55:25 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.48999999999999999,"15min":0.48599999999999999},"min":{"1min":0.377,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.504,"5min":0.501,"15min":0.501},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.76500000000000001,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.30499999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574912,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298068,"kb_used_data":708,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447084,"statfs":{"total":95995035648,"available":95689814016,"internally_reserved":0,"allocated":724992,"data_stored":316469,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:55:09 2024","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.36199999999999999,"15min":0.32700000000000001},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.373,"15min":0.33500000000000002},"min":{"1min":0.27600000000000002,"5min":0.214,"15min":0.214},"max":{"1min":0.62,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.40000000000000002}]},{"osd":2,"last update":"Fri Apr 26 04:55:29 2024","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.40000000000000002,"15min":0.38800000000000001},"min":{"1min":0.29099999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.42199999999999999,"15min":0.41599999999999998},"min":{"1min":0.28999999999999998,"5min":0.22,"15min":0.22},"max":{"1min":0.62,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.36799999999999999}]},{"osd":3,"last update":"Fri Apr 26 04:55:58 2024","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.40899999999999997,"15min":0.41099999999999998},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.52700000000000002,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.314},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.46899999999999997,"15min":0.47899999999999998},"min":{"1min":0.32100000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.60699999999999998,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.41899999999999998}]},{"osd":4,"last update":"Fri Apr 26 04:55:16 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.56200000000000006,"15min":0.54700000000000004},"min":{"1min":0.41399999999999998,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.77700000000000002,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.55000000000000004,"15min":0.53400000000000003},"min":{"1min":0.42799999999999999,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.79300000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.45900000000000002}]},{"osd":5,"last update":"Fri Apr 26 04:55:45 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.60299999999999998,"15min":0.59899999999999998},"min":{"1min":0.41499999999999998,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.82399999999999995,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.60799999999999998,"15min":0.60899999999999999},"min":{"1min":0.44,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.77300000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.44800000000000001}]},{"osd":6,"last update":"Fri Apr 26 04:55:04 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.64800000000000002,"15min":0.65200000000000002},"min":{"1min":0.42599999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.878,"5min":1.0269999999999999,"15min":1.0269999999999999},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.64300000000000002,"5min":1.6539999999999999,"15min":1.8220000000000001},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.80800000000000005,"5min":24.962,"15min":24.962},"last":0.48699999999999999}]},{"osd":7,"last update":"Fri Apr 26 04:55:25 2024","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.84599999999999997,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.65500000000000003,"15min":0.65700000000000003},"min":{"1min":0.498,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.81799999999999995,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.55800000000000005}]}]},{"osd":0,"up_from":8,"seq":34359738436,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298652,"kb_used_data":1292,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446500,"statfs":{"total":95995035648,"available":95689216000,"internally_reserved":0,"allocated":1323008,"data_stored":907164,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 26 04:55:10 2024","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.39100000000000001,"15min":0.36799999999999999},"min":{"1min":0.25800000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.34999999999999998,"5min":0.39300000000000002,"15min":0.38300000000000001},"min":{"1min":0.26800000000000002,"5min":0.22,"15min":0.22},"max":{"1min":0.51200000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.39900000000000002}]},{"osd":2,"last update":"Fri Apr 26 04:55:39 2024","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.40600000000000003,"15min":0.40400000000000003},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.53500000000000003,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.45100000000000001,"15min":0.45600000000000002},"min":{"1min":0.26500000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.63600000000000001}]},{"osd":3,"last update":"Fri Apr 26 04:55:55 2024","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.46500000000000002,"15min":0.48299999999999998},"min":{"1min":0.33900000000000002,"5min":0.249,"15min":0.249},"max":{"1min":1.1140000000000001,"5min":1.1140000000000001,"15min":1.1140000000000001},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.497,"15min":0.50800000000000001},"min":{"1min":0.36199999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.0209999999999999,"5min":1.0209999999999999,"15min":1.0209999999999999},"last":0.58699999999999997}]},{"osd":4,"last update":"Fri Apr 26 04:55:17 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.623,"15min":0.63700000000000001},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.81299999999999994},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59999999999999998,"15min":0.60599999999999998},"min":{"1min":0.44800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.82699999999999996,"5min":0.872,"15min":0.872},"last":0.79500000000000004}]},{"osd":5,"last update":"Fri Apr 26 04:55:39 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.60999999999999999,"15min":0.62},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.76200000000000001,"5min":0.88,"15min":0.88},"last":0.84299999999999997},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.626,"15min":0.64200000000000002},"min":{"1min":0.41899999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.79100000000000004,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.82799999999999996}]},{"osd":6,"last update":"Fri Apr 26 04:56:01 2024","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63200000000000001,"15min":0.64000000000000001},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.0669999999999999,"5min":1.0669999999999999,"15min":1.0669999999999999},"last":0.76200000000000001},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.66500000000000004,"15min":0.67500000000000004},"min":{"1min":0.46600000000000003,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.88800000000000001,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.74199999999999999}]},{"osd":7,"last update":"Fri Apr 26 04:55:26 2024","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.65800000000000003,"15min":0.66100000000000003},"min":{"1min":0.48899999999999999,"5min":0.434,"15min":0.434},"max":{"1min":0.95899999999999996,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.77700000000000002},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.63,"15min":0.63200000000000001},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.77700000000000002,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.71899999999999997}]}]},{"osd":2,"up_from":18,"seq":77309411387,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298072,"kb_used_data":716,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447080,"statfs":{"total":95995035648,"available":95689809920,"internally_reserved":0,"allocated":733184,"data_stored":317073,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:55:31 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.38,"15min":0.38600000000000001},"min":{"1min":0.26400000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.57499999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.45700000000000002,"15min":0.441},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.61399999999999999,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.33500000000000002}]},{"osd":1,"last update":"Fri Apr 26 04:55:31 2024","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.40799999999999997,"15min":0.41199999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.434},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.42599999999999999,"15min":0.42399999999999999},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.53900000000000003,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.371}]},{"osd":3,"last update":"Fri Apr 26 04:55:59 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.47099999999999997,"15min":0.48099999999999998},"min":{"1min":0.307,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.64000000000000001,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.307},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.49399999999999999,"15min":0.51400000000000001},"min":{"1min":0.33100000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.72099999999999997,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.41499999999999998}]},{"osd":4,"last update":"Fri Apr 26 04:55:16 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.58399999999999996,"15min":0.59499999999999997},"min":{"1min":0.40300000000000002,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.79900000000000004,"5min":1.042,"15min":1.042},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.61599999999999999,"15min":0.628},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.72699999999999998,"5min":1.1240000000000001,"15min":1.1240000000000001},"last":0.52800000000000002}]},{"osd":5,"last update":"Fri Apr 26 04:55:40 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.64200000000000002,"15min":0.66600000000000004},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.80700000000000005,"5min":1.077,"15min":1.077},"last":0.505},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.61499999999999999,"15min":0.627},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.89200000000000002,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.54900000000000004}]},{"osd":6,"last update":"Fri Apr 26 04:55:59 2024","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.65000000000000002,"15min":0.66400000000000003},"min":{"1min":0.45100000000000001,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.876,"5min":1.012,"15min":1.012},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.65900000000000003,"15min":0.68100000000000005},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.93000000000000005,"5min":1.0960000000000001,"15min":1.0960000000000001},"last":0.51200000000000001}]},{"osd":7,"last update":"Fri Apr 26 04:55:22 2024","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.65900000000000003,"15min":0.66100000000000003},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.81200000000000006,"5min":0.998,"15min":0.998},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.626,"15min":0.628},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.85699999999999998,"5min":0.875,"15min":0.875},"last":0.57999999999999996}]}]},{"osd":3,"up_from":26,"seq":111669149749,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298072,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447080,"statfs":{"total":95995035648,"available":95689809920,"internally_reserved":0,"allocated":729088,"data_stored":316559,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:55:57 2024","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.42799999999999999,"15min":0.45500000000000002},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.48199999999999998,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.42099999999999999,"15min":0.44900000000000001},"min":{"1min":0.26300000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.34599999999999997}]},{"osd":1,"last update":"Fri Apr 26 04:55:57 2024","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.45300000000000001,"15min":0.48299999999999998},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.56899999999999995,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.44400000000000001,"15min":0.47499999999999998},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.59299999999999997,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.38300000000000001}]},{"osd":2,"last update":"Fri Apr 26 04:55:57 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.48199999999999998,"15min":0.499},"min":{"1min":0.34999999999999998,"5min":0.317,"15min":0.317},"max":{"1min":0.57299999999999995,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.45400000000000001,"15min":0.498},"min":{"1min":0.29199999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.55300000000000005,"5min":0.748,"15min":0.748},"last":0.437}]},{"osd":4,"last update":"Fri Apr 26 04:55:17 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.61099999999999999,"15min":0.61799999999999999},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59299999999999997,"15min":0.60099999999999998},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.56899999999999995}]},{"osd":5,"last update":"Fri Apr 26 04:55:41 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":1.1160000000000001,"15min":1.371},"min":{"1min":0.41299999999999998,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.86299999999999999,"5min":18.239000000000001,"15min":18.239000000000001},"last":0.62},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.65000000000000002,"15min":0.65800000000000003},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.82899999999999996,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.53700000000000003}]},{"osd":6,"last update":"Fri Apr 26 04:56:00 2024","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":1.2350000000000001,"15min":1.5029999999999999},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.94299999999999995,"5min":18.221,"15min":18.221},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.52200000000000002}]},{"osd":7,"last update":"Fri Apr 26 04:55:26 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.56399999999999995,"15min":0.55600000000000005},"min":{"1min":0.45700000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.69699999999999995,"5min":0.67600000000000005,"15min":0.67200000000000004},"min":{"1min":0.53600000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.66100000000000003}]}]},{"osd":4,"up_from":31,"seq":133143986225,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298092,"kb_used_data":732,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447060,"statfs":{"total":95995035648,"available":95689789440,"internally_reserved":0,"allocated":749568,"data_stored":318167,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:55:15 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.502,"15min":0.49399999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.746,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.52500000000000002,"15min":0.52200000000000002},"min":{"1min":0.316,"5min":0.309,"15min":0.309},"max":{"1min":0.80900000000000005,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.74099999999999999}]},{"osd":1,"last update":"Fri Apr 26 04:55:15 2024","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":0.55300000000000005,"15min":0.51500000000000001},"min":{"1min":0.33900000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":4.3700000000000001,"5min":4.3700000000000001,"15min":4.3700000000000001},"last":0.624},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.51700000000000002,"15min":0.51200000000000001},"min":{"1min":0.34399999999999997,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.68400000000000005}]},{"osd":2,"last update":"Fri Apr 26 04:55:15 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.52400000000000002,"15min":0.52900000000000003},"min":{"1min":0.372,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.496,"5min":0.50600000000000001,"15min":0.501},"min":{"1min":0.36299999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.65800000000000003,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.66700000000000004}]},{"osd":3,"last update":"Fri Apr 26 04:55:15 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.54300000000000004,"15min":0.53900000000000003},"min":{"1min":0.41299999999999998,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.71599999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.54900000000000004,"15min":0.53900000000000003},"min":{"1min":0.42599999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.78700000000000003,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.69599999999999995}]},{"osd":5,"last update":"Fri Apr 26 04:55:46 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.41299999999999998,"15min":0.39100000000000001},"min":{"1min":0.33100000000000002,"5min":0.221,"15min":0.221},"max":{"1min":0.61099999999999999,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.42099999999999999,"15min":0.39800000000000002},"min":{"1min":0.34699999999999998,"5min":0.251,"15min":0.251},"max":{"1min":0.54600000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.53000000000000003}]},{"osd":6,"last update":"Fri Apr 26 04:55:02 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.46600000000000003,"15min":0.46000000000000002},"min":{"1min":0.315,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.65400000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.502},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.33600000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.71699999999999997,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.57799999999999996}]},{"osd":7,"last update":"Fri Apr 26 04:55:25 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.52500000000000002,"15min":0.52200000000000002},"min":{"1min":0.432,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.69999999999999996,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.52300000000000002,"15min":0.52100000000000002},"min":{"1min":0.40200000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.67100000000000004,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.56200000000000006}]}]},{"osd":5,"up_from":39,"seq":167503724589,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298068,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447084,"statfs":{"total":95995035648,"available":95689814016,"internally_reserved":0,"allocated":729088,"data_stored":316786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:55:37 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.53500000000000003,"15min":0.54800000000000004},"min":{"1min":0.32600000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.70299999999999996,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.628},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.55100000000000005,"15min":0.55700000000000005},"min":{"1min":0.41399999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.68200000000000005,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.57899999999999996}]},{"osd":1,"last update":"Fri Apr 26 04:55:37 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.53900000000000003,"15min":0.54900000000000004},"min":{"1min":0.40300000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.627,"5min":0.75,"15min":0.75},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.55200000000000005,"15min":0.56299999999999994},"min":{"1min":0.373,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.73499999999999999,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.61599999999999999}]},{"osd":2,"last update":"Fri Apr 26 04:55:37 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55600000000000005,"15min":0.56399999999999995},"min":{"1min":0.40600000000000003,"5min":0.317,"15min":0.317},"max":{"1min":0.72299999999999998,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.57199999999999995,"15min":0.58999999999999997},"min":{"1min":0.36699999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.69299999999999995,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.52300000000000002}]},{"osd":3,"last update":"Fri Apr 26 04:55:37 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.60399999999999998,"15min":0.62},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.72499999999999998,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.59699999999999998,"15min":0.60699999999999998},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.69099999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.56599999999999995}]},{"osd":4,"last update":"Fri Apr 26 04:55:37 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.41299999999999998,"15min":0.39200000000000002},"min":{"1min":0.32000000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.433,"15min":0.40500000000000003},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.78100000000000003,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.59399999999999997}]},{"osd":6,"last update":"Fri Apr 26 04:55:02 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.44500000000000001,"15min":0.438},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.501,"15min":0.49299999999999999},"min":{"1min":0.34799999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.54700000000000004}]},{"osd":7,"last update":"Fri Apr 26 04:55:25 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.47499999999999998,"15min":0.46899999999999997},"min":{"1min":0.33700000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.751,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.439},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.51400000000000001,"15min":0.51100000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.503}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-26T04:56:07.305 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-26T04:56:07.579 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:07.778 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:07 smithi012 ceph-mon[26617]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:07.778 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:07 smithi012 ceph-mon[26617]: from='client.24686 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:56:07.778 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:07 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:07.778 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:07 smithi012 ceph-mon[37567]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:07.778 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:07 smithi012 ceph-mon[37567]: from='client.24686 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:56:07.778 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:07 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:07.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:07 smithi116 ceph-mon[29897]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:07.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:07 smithi116 ceph-mon[29897]: from='client.24686 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:56:07.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:07 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:09.527 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:56:09.528 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2024-04-26T04:56:09.841 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:09 smithi012 ceph-mon[26617]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:09.841 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:09 smithi012 ceph-mon[37567]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:09.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:09 smithi116 ceph-mon[29897]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:10.170 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":82,"stamp":"2024-04-26T04:56:08.431710+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386348,"kb_used_data":7480,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747574868,"statfs":{"total":767960285184,"available":765516664832,"internally_reserved":0,"allocated":7659520,"data_stored":4308421,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002710"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.497927+0000","last_change":"2024-04-26T04:55:07.502061+0000","last_active":"2024-04-26T04:55:52.497927+0000","last_peered":"2024-04-26T04:55:52.497927+0000","last_clean":"2024-04-26T04:55:52.497927+0000","last_became_active":"2024-04-26T04:55:07.501751+0000","last_became_peered":"2024-04-26T04:55:07.501751+0000","last_unstale":"2024-04-26T04:55:52.497927+0000","last_undegraded":"2024-04-26T04:55:52.497927+0000","last_fullsized":"2024-04-26T04:55:52.497927+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:18:00.822676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.820585+0000","last_change":"2024-04-26T04:55:07.456603+0000","last_active":"2024-04-26T04:55:52.820585+0000","last_peered":"2024-04-26T04:55:52.820585+0000","last_clean":"2024-04-26T04:55:52.820585+0000","last_became_active":"2024-04-26T04:55:07.456287+0000","last_became_peered":"2024-04-26T04:55:07.456287+0000","last_unstale":"2024-04-26T04:55:52.820585+0000","last_undegraded":"2024-04-26T04:55:52.820585+0000","last_fullsized":"2024-04-26T04:55:52.820585+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:48:36.315805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498785+0000","last_change":"2024-04-26T04:55:08.476806+0000","last_active":"2024-04-26T04:55:52.498785+0000","last_peered":"2024-04-26T04:55:52.498785+0000","last_clean":"2024-04-26T04:55:52.498785+0000","last_became_active":"2024-04-26T04:55:08.476506+0000","last_became_peered":"2024-04-26T04:55:08.476506+0000","last_unstale":"2024-04-26T04:55:52.498785+0000","last_undegraded":"2024-04-26T04:55:52.498785+0000","last_fullsized":"2024-04-26T04:55:52.498785+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:55:21.535316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.499927+0000","last_change":"2024-04-26T04:55:07.456419+0000","last_active":"2024-04-26T04:55:52.499927+0000","last_peered":"2024-04-26T04:55:52.499927+0000","last_clean":"2024-04-26T04:55:52.499927+0000","last_became_active":"2024-04-26T04:55:07.455639+0000","last_became_peered":"2024-04-26T04:55:07.455639+0000","last_unstale":"2024-04-26T04:55:52.499927+0000","last_undegraded":"2024-04-26T04:55:52.499927+0000","last_fullsized":"2024-04-26T04:55:52.499927+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:11:53.038126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498508+0000","last_change":"2024-04-26T04:55:07.482572+0000","last_active":"2024-04-26T04:55:52.498508+0000","last_peered":"2024-04-26T04:55:52.498508+0000","last_clean":"2024-04-26T04:55:52.498508+0000","last_became_active":"2024-04-26T04:55:07.482371+0000","last_became_peered":"2024-04-26T04:55:07.482371+0000","last_unstale":"2024-04-26T04:55:52.498508+0000","last_undegraded":"2024-04-26T04:55:52.498508+0000","last_fullsized":"2024-04-26T04:55:52.498508+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:00:03.465526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.496543+0000","last_change":"2024-04-26T04:55:07.445793+0000","last_active":"2024-04-26T04:55:52.496543+0000","last_peered":"2024-04-26T04:55:52.496543+0000","last_clean":"2024-04-26T04:55:52.496543+0000","last_became_active":"2024-04-26T04:55:07.445496+0000","last_became_peered":"2024-04-26T04:55:07.445496+0000","last_unstale":"2024-04-26T04:55:52.496543+0000","last_undegraded":"2024-04-26T04:55:52.496543+0000","last_fullsized":"2024-04-26T04:55:52.496543+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:53:56.081693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-26T04:55:53.687991+0000","last_change":"2024-04-26T04:55:53.687991+0000","last_active":"2024-04-26T04:55:53.687991+0000","last_peered":"2024-04-26T04:55:53.687991+0000","last_clean":"2024-04-26T04:55:53.687991+0000","last_became_active":"2024-04-26T04:55:53.687560+0000","last_became_peered":"2024-04-26T04:55:53.687560+0000","last_unstale":"2024-04-26T04:55:53.687991+0000","last_undegraded":"2024-04-26T04:55:53.687991+0000","last_fullsized":"2024-04-26T04:55:53.687991+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:42:42.995605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.496269+0000","last_change":"2024-04-26T04:55:07.449303+0000","last_active":"2024-04-26T04:55:52.496269+0000","last_peered":"2024-04-26T04:55:52.496269+0000","last_clean":"2024-04-26T04:55:52.496269+0000","last_became_active":"2024-04-26T04:55:07.449039+0000","last_became_peered":"2024-04-26T04:55:07.449039+0000","last_unstale":"2024-04-26T04:55:52.496269+0000","last_undegraded":"2024-04-26T04:55:52.496269+0000","last_fullsized":"2024-04-26T04:55:52.496269+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:49:20.436815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.499259+0000","last_change":"2024-04-26T04:55:07.501687+0000","last_active":"2024-04-26T04:55:52.499259+0000","last_peered":"2024-04-26T04:55:52.499259+0000","last_clean":"2024-04-26T04:55:52.499259+0000","last_became_active":"2024-04-26T04:55:07.501318+0000","last_became_peered":"2024-04-26T04:55:07.501318+0000","last_unstale":"2024-04-26T04:55:52.499259+0000","last_undegraded":"2024-04-26T04:55:52.499259+0000","last_fullsized":"2024-04-26T04:55:52.499259+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:41:42.194849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.820456+0000","last_change":"2024-04-26T04:55:07.461008+0000","last_active":"2024-04-26T04:55:52.820456+0000","last_peered":"2024-04-26T04:55:52.820456+0000","last_clean":"2024-04-26T04:55:52.820456+0000","last_became_active":"2024-04-26T04:55:07.458834+0000","last_became_peered":"2024-04-26T04:55:07.458834+0000","last_unstale":"2024-04-26T04:55:52.820456+0000","last_undegraded":"2024-04-26T04:55:52.820456+0000","last_fullsized":"2024-04-26T04:55:52.820456+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:30:44.154385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498584+0000","last_change":"2024-04-26T04:55:07.474089+0000","last_active":"2024-04-26T04:55:52.498584+0000","last_peered":"2024-04-26T04:55:52.498584+0000","last_clean":"2024-04-26T04:55:52.498584+0000","last_became_active":"2024-04-26T04:55:07.473624+0000","last_became_peered":"2024-04-26T04:55:07.473624+0000","last_unstale":"2024-04-26T04:55:52.498584+0000","last_undegraded":"2024-04-26T04:55:52.498584+0000","last_fullsized":"2024-04-26T04:55:52.498584+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:15:47.507462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.820479+0000","last_change":"2024-04-26T04:55:07.458133+0000","last_active":"2024-04-26T04:55:52.820479+0000","last_peered":"2024-04-26T04:55:52.820479+0000","last_clean":"2024-04-26T04:55:52.820479+0000","last_became_active":"2024-04-26T04:55:07.457777+0000","last_became_peered":"2024-04-26T04:55:07.457777+0000","last_unstale":"2024-04-26T04:55:52.820479+0000","last_undegraded":"2024-04-26T04:55:52.820479+0000","last_fullsized":"2024-04-26T04:55:52.820479+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:21:53.116783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498998+0000","last_change":"2024-04-26T04:53:57.857083+0000","last_active":"2024-04-26T04:55:52.498998+0000","last_peered":"2024-04-26T04:55:52.498998+0000","last_clean":"2024-04-26T04:55:52.498998+0000","last_became_active":"2024-04-26T04:53:57.856775+0000","last_became_peered":"2024-04-26T04:53:57.856775+0000","last_unstale":"2024-04-26T04:55:52.498998+0000","last_undegraded":"2024-04-26T04:55:52.498998+0000","last_fullsized":"2024-04-26T04:55:52.498998+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:06:53.087376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498696+0000","last_change":"2024-04-26T04:55:08.601953+0000","last_active":"2024-04-26T04:55:52.498696+0000","last_peered":"2024-04-26T04:55:52.498696+0000","last_clean":"2024-04-26T04:55:52.498696+0000","last_became_active":"2024-04-26T04:55:08.601525+0000","last_became_peered":"2024-04-26T04:55:08.601525+0000","last_unstale":"2024-04-26T04:55:52.498696+0000","last_undegraded":"2024-04-26T04:55:52.498696+0000","last_fullsized":"2024-04-26T04:55:52.498696+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:28:47.387070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.496057+0000","last_change":"2024-04-26T04:53:57.938166+0000","last_active":"2024-04-26T04:55:52.496057+0000","last_peered":"2024-04-26T04:55:52.496057+0000","last_clean":"2024-04-26T04:55:52.496057+0000","last_became_active":"2024-04-26T04:53:57.937842+0000","last_became_peered":"2024-04-26T04:53:57.937842+0000","last_unstale":"2024-04-26T04:55:52.496057+0000","last_undegraded":"2024-04-26T04:55:52.496057+0000","last_fullsized":"2024-04-26T04:55:52.496057+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:39:45.320158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.500094+0000","last_change":"2024-04-26T04:55:07.464082+0000","last_active":"2024-04-26T04:55:52.500094+0000","last_peered":"2024-04-26T04:55:52.500094+0000","last_clean":"2024-04-26T04:55:52.500094+0000","last_became_active":"2024-04-26T04:55:07.463779+0000","last_became_peered":"2024-04-26T04:55:07.463779+0000","last_unstale":"2024-04-26T04:55:52.500094+0000","last_undegraded":"2024-04-26T04:55:52.500094+0000","last_fullsized":"2024-04-26T04:55:52.500094+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:08:24.246321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.820500+0000","last_change":"2024-04-26T04:55:08.603346+0000","last_active":"2024-04-26T04:55:52.820500+0000","last_peered":"2024-04-26T04:55:52.820500+0000","last_clean":"2024-04-26T04:55:52.820500+0000","last_became_active":"2024-04-26T04:55:08.603046+0000","last_became_peered":"2024-04-26T04:55:08.603046+0000","last_unstale":"2024-04-26T04:55:52.820500+0000","last_undegraded":"2024-04-26T04:55:52.820500+0000","last_fullsized":"2024-04-26T04:55:52.820500+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:50:37.364418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.495774+0000","last_change":"2024-04-26T04:55:07.450160+0000","last_active":"2024-04-26T04:55:52.495774+0000","last_peered":"2024-04-26T04:55:52.495774+0000","last_clean":"2024-04-26T04:55:52.495774+0000","last_became_active":"2024-04-26T04:55:07.449749+0000","last_became_peered":"2024-04-26T04:55:07.449749+0000","last_unstale":"2024-04-26T04:55:52.495774+0000","last_undegraded":"2024-04-26T04:55:52.495774+0000","last_fullsized":"2024-04-26T04:55:52.495774+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:22:48.334435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.501748+0000","last_change":"2024-04-26T04:53:57.939093+0000","last_active":"2024-04-26T04:55:52.501748+0000","last_peered":"2024-04-26T04:55:52.501748+0000","last_clean":"2024-04-26T04:55:52.501748+0000","last_became_active":"2024-04-26T04:53:57.938662+0000","last_became_peered":"2024-04-26T04:53:57.938662+0000","last_unstale":"2024-04-26T04:55:52.501748+0000","last_undegraded":"2024-04-26T04:55:52.501748+0000","last_fullsized":"2024-04-26T04:55:52.501748+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:37:22.972048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.820510+0000","last_change":"2024-04-26T04:55:08.476125+0000","last_active":"2024-04-26T04:55:52.820510+0000","last_peered":"2024-04-26T04:55:52.820510+0000","last_clean":"2024-04-26T04:55:52.820510+0000","last_became_active":"2024-04-26T04:55:08.475766+0000","last_became_peered":"2024-04-26T04:55:08.475766+0000","last_unstale":"2024-04-26T04:55:52.820510+0000","last_undegraded":"2024-04-26T04:55:52.820510+0000","last_fullsized":"2024-04-26T04:55:52.820510+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:46:44.083324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.820305+0000","last_change":"2024-04-26T04:55:07.453608+0000","last_active":"2024-04-26T04:55:52.820305+0000","last_peered":"2024-04-26T04:55:52.820305+0000","last_clean":"2024-04-26T04:55:52.820305+0000","last_became_active":"2024-04-26T04:55:07.453310+0000","last_became_peered":"2024-04-26T04:55:07.453310+0000","last_unstale":"2024-04-26T04:55:52.820305+0000","last_undegraded":"2024-04-26T04:55:52.820305+0000","last_fullsized":"2024-04-26T04:55:52.820305+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:33:40.865186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.501473+0000","last_change":"2024-04-26T04:53:57.939087+0000","last_active":"2024-04-26T04:55:52.501473+0000","last_peered":"2024-04-26T04:55:52.501473+0000","last_clean":"2024-04-26T04:55:52.501473+0000","last_became_active":"2024-04-26T04:53:57.938647+0000","last_became_peered":"2024-04-26T04:53:57.938647+0000","last_unstale":"2024-04-26T04:55:52.501473+0000","last_undegraded":"2024-04-26T04:55:52.501473+0000","last_fullsized":"2024-04-26T04:55:52.501473+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:51:38.687896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.499065+0000","last_change":"2024-04-26T04:55:07.500866+0000","last_active":"2024-04-26T04:55:52.499065+0000","last_peered":"2024-04-26T04:55:52.499065+0000","last_clean":"2024-04-26T04:55:52.499065+0000","last_became_active":"2024-04-26T04:55:07.500516+0000","last_became_peered":"2024-04-26T04:55:07.500516+0000","last_unstale":"2024-04-26T04:55:52.499065+0000","last_undegraded":"2024-04-26T04:55:52.499065+0000","last_fullsized":"2024-04-26T04:55:52.499065+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:10:30.230397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.499274+0000","last_change":"2024-04-26T04:55:07.456313+0000","last_active":"2024-04-26T04:55:52.499274+0000","last_peered":"2024-04-26T04:55:52.499274+0000","last_clean":"2024-04-26T04:55:52.499274+0000","last_became_active":"2024-04-26T04:55:07.456037+0000","last_became_peered":"2024-04-26T04:55:07.456037+0000","last_unstale":"2024-04-26T04:55:52.499274+0000","last_undegraded":"2024-04-26T04:55:52.499274+0000","last_fullsized":"2024-04-26T04:55:52.499274+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:57:09.971101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":278,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.501397+0000","last_change":"2024-04-26T04:53:25.550145+0000","last_active":"2024-04-26T04:55:52.501397+0000","last_peered":"2024-04-26T04:55:52.501397+0000","last_clean":"2024-04-26T04:55:52.501397+0000","last_became_active":"2024-04-26T04:53:25.549600+0000","last_became_peered":"2024-04-26T04:53:25.549600+0000","last_unstale":"2024-04-26T04:55:52.501397+0000","last_undegraded":"2024-04-26T04:55:52.501397+0000","last_fullsized":"2024-04-26T04:55:52.501397+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:51:27.309540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:51:27.309540+0000","last_clean_scrub_stamp":"2024-04-26T04:51:27.309540+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:23:44.865329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.496364+0000","last_change":"2024-04-26T04:53:57.856241+0000","last_active":"2024-04-26T04:55:52.496364+0000","last_peered":"2024-04-26T04:55:52.496364+0000","last_clean":"2024-04-26T04:55:52.496364+0000","last_became_active":"2024-04-26T04:53:57.855745+0000","last_became_peered":"2024-04-26T04:53:57.855745+0000","last_unstale":"2024-04-26T04:55:52.496364+0000","last_undegraded":"2024-04-26T04:55:52.496364+0000","last_fullsized":"2024-04-26T04:55:52.496364+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:54:03.208950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.496217+0000","last_change":"2024-04-26T04:55:08.478015+0000","last_active":"2024-04-26T04:55:52.496217+0000","last_peered":"2024-04-26T04:55:52.496217+0000","last_clean":"2024-04-26T04:55:52.496217+0000","last_became_active":"2024-04-26T04:55:08.477711+0000","last_became_peered":"2024-04-26T04:55:08.477711+0000","last_unstale":"2024-04-26T04:55:52.496217+0000","last_undegraded":"2024-04-26T04:55:52.496217+0000","last_fullsized":"2024-04-26T04:55:52.496217+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T04:57:44.192610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.495349+0000","last_change":"2024-04-26T04:55:07.452343+0000","last_active":"2024-04-26T04:55:52.495349+0000","last_peered":"2024-04-26T04:55:52.495349+0000","last_clean":"2024-04-26T04:55:52.495349+0000","last_became_active":"2024-04-26T04:55:07.452057+0000","last_became_peered":"2024-04-26T04:55:07.452057+0000","last_unstale":"2024-04-26T04:55:52.495349+0000","last_undegraded":"2024-04-26T04:55:52.495349+0000","last_fullsized":"2024-04-26T04:55:52.495349+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:01:49.065366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498813+0000","last_change":"2024-04-26T04:53:57.939187+0000","last_active":"2024-04-26T04:55:52.498813+0000","last_peered":"2024-04-26T04:55:52.498813+0000","last_clean":"2024-04-26T04:55:52.498813+0000","last_became_active":"2024-04-26T04:53:57.938749+0000","last_became_peered":"2024-04-26T04:53:57.938749+0000","last_unstale":"2024-04-26T04:55:52.498813+0000","last_undegraded":"2024-04-26T04:55:52.498813+0000","last_fullsized":"2024-04-26T04:55:52.498813+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:59:54.974432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.495694+0000","last_change":"2024-04-26T04:55:08.474447+0000","last_active":"2024-04-26T04:55:52.495694+0000","last_peered":"2024-04-26T04:55:52.495694+0000","last_clean":"2024-04-26T04:55:52.495694+0000","last_became_active":"2024-04-26T04:55:08.474159+0000","last_became_peered":"2024-04-26T04:55:08.474159+0000","last_unstale":"2024-04-26T04:55:52.495694+0000","last_undegraded":"2024-04-26T04:55:52.495694+0000","last_fullsized":"2024-04-26T04:55:52.495694+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:18:45.625120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498169+0000","last_change":"2024-04-26T04:55:07.457209+0000","last_active":"2024-04-26T04:55:52.498169+0000","last_peered":"2024-04-26T04:55:52.498169+0000","last_clean":"2024-04-26T04:55:52.498169+0000","last_became_active":"2024-04-26T04:55:07.456997+0000","last_became_peered":"2024-04-26T04:55:07.456997+0000","last_unstale":"2024-04-26T04:55:52.498169+0000","last_undegraded":"2024-04-26T04:55:52.498169+0000","last_fullsized":"2024-04-26T04:55:52.498169+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:27:28.288028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.496164+0000","last_change":"2024-04-26T04:53:57.856053+0000","last_active":"2024-04-26T04:55:52.496164+0000","last_peered":"2024-04-26T04:55:52.496164+0000","last_clean":"2024-04-26T04:55:52.496164+0000","last_became_active":"2024-04-26T04:53:57.855693+0000","last_became_peered":"2024-04-26T04:53:57.855693+0000","last_unstale":"2024-04-26T04:55:52.496164+0000","last_undegraded":"2024-04-26T04:55:52.496164+0000","last_fullsized":"2024-04-26T04:55:52.496164+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:35:20.404397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.495422+0000","last_change":"2024-04-26T04:55:08.569282+0000","last_active":"2024-04-26T04:55:52.495422+0000","last_peered":"2024-04-26T04:55:52.495422+0000","last_clean":"2024-04-26T04:55:52.495422+0000","last_became_active":"2024-04-26T04:55:08.568536+0000","last_became_peered":"2024-04-26T04:55:08.568536+0000","last_unstale":"2024-04-26T04:55:52.495422+0000","last_undegraded":"2024-04-26T04:55:52.495422+0000","last_fullsized":"2024-04-26T04:55:52.495422+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:55:18.326130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.496511+0000","last_change":"2024-04-26T04:55:07.448584+0000","last_active":"2024-04-26T04:55:52.496511+0000","last_peered":"2024-04-26T04:55:52.496511+0000","last_clean":"2024-04-26T04:55:52.496511+0000","last_became_active":"2024-04-26T04:55:07.447767+0000","last_became_peered":"2024-04-26T04:55:07.447767+0000","last_unstale":"2024-04-26T04:55:52.496511+0000","last_undegraded":"2024-04-26T04:55:52.496511+0000","last_fullsized":"2024-04-26T04:55:52.496511+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:51:17.365374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.495618+0000","last_change":"2024-04-26T04:53:57.939812+0000","last_active":"2024-04-26T04:55:52.495618+0000","last_peered":"2024-04-26T04:55:52.495618+0000","last_clean":"2024-04-26T04:55:52.495618+0000","last_became_active":"2024-04-26T04:53:57.939509+0000","last_became_peered":"2024-04-26T04:53:57.939509+0000","last_unstale":"2024-04-26T04:55:52.495618+0000","last_undegraded":"2024-04-26T04:55:52.495618+0000","last_fullsized":"2024-04-26T04:55:52.495618+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:05:41.872666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.496143+0000","last_change":"2024-04-26T04:55:08.568905+0000","last_active":"2024-04-26T04:55:52.496143+0000","last_peered":"2024-04-26T04:55:52.496143+0000","last_clean":"2024-04-26T04:55:52.496143+0000","last_became_active":"2024-04-26T04:55:08.568603+0000","last_became_peered":"2024-04-26T04:55:08.568603+0000","last_unstale":"2024-04-26T04:55:52.496143+0000","last_undegraded":"2024-04-26T04:55:52.496143+0000","last_fullsized":"2024-04-26T04:55:52.496143+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:14:11.351749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498784+0000","last_change":"2024-04-26T04:55:07.457164+0000","last_active":"2024-04-26T04:55:52.498784+0000","last_peered":"2024-04-26T04:55:52.498784+0000","last_clean":"2024-04-26T04:55:52.498784+0000","last_became_active":"2024-04-26T04:55:07.456826+0000","last_became_peered":"2024-04-26T04:55:07.456826+0000","last_unstale":"2024-04-26T04:55:52.498784+0000","last_undegraded":"2024-04-26T04:55:52.498784+0000","last_fullsized":"2024-04-26T04:55:52.498784+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:00:42.598533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498791+0000","last_change":"2024-04-26T04:55:07.502223+0000","last_active":"2024-04-26T04:55:52.498791+0000","last_peered":"2024-04-26T04:55:52.498791+0000","last_clean":"2024-04-26T04:55:52.498791+0000","last_became_active":"2024-04-26T04:55:07.501930+0000","last_became_peered":"2024-04-26T04:55:07.501930+0000","last_unstale":"2024-04-26T04:55:52.498791+0000","last_undegraded":"2024-04-26T04:55:52.498791+0000","last_fullsized":"2024-04-26T04:55:52.498791+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:01:41.107020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498392+0000","last_change":"2024-04-26T04:55:07.451911+0000","last_active":"2024-04-26T04:55:52.498392+0000","last_peered":"2024-04-26T04:55:52.498392+0000","last_clean":"2024-04-26T04:55:52.498392+0000","last_became_active":"2024-04-26T04:55:07.451612+0000","last_became_peered":"2024-04-26T04:55:07.451612+0000","last_unstale":"2024-04-26T04:55:52.498392+0000","last_undegraded":"2024-04-26T04:55:52.498392+0000","last_fullsized":"2024-04-26T04:55:52.498392+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:38:48.221948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.499125+0000","last_change":"2024-04-26T04:55:07.502561+0000","last_active":"2024-04-26T04:55:52.499125+0000","last_peered":"2024-04-26T04:55:52.499125+0000","last_clean":"2024-04-26T04:55:52.499125+0000","last_became_active":"2024-04-26T04:55:07.501680+0000","last_became_peered":"2024-04-26T04:55:07.501680+0000","last_unstale":"2024-04-26T04:55:52.499125+0000","last_undegraded":"2024-04-26T04:55:52.499125+0000","last_fullsized":"2024-04-26T04:55:52.499125+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:56:56.062544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.499947+0000","last_change":"2024-04-26T04:55:07.450789+0000","last_active":"2024-04-26T04:55:52.499947+0000","last_peered":"2024-04-26T04:55:52.499947+0000","last_clean":"2024-04-26T04:55:52.499947+0000","last_became_active":"2024-04-26T04:55:07.450560+0000","last_became_peered":"2024-04-26T04:55:07.450560+0000","last_unstale":"2024-04-26T04:55:52.499947+0000","last_undegraded":"2024-04-26T04:55:52.499947+0000","last_fullsized":"2024-04-26T04:55:52.499947+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:10:49.656045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498227+0000","last_change":"2024-04-26T04:55:07.474296+0000","last_active":"2024-04-26T04:55:52.498227+0000","last_peered":"2024-04-26T04:55:52.498227+0000","last_clean":"2024-04-26T04:55:52.498227+0000","last_became_active":"2024-04-26T04:55:07.473638+0000","last_became_peered":"2024-04-26T04:55:07.473638+0000","last_unstale":"2024-04-26T04:55:52.498227+0000","last_undegraded":"2024-04-26T04:55:52.498227+0000","last_fullsized":"2024-04-26T04:55:52.498227+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:57:58.375045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.499334+0000","last_change":"2024-04-26T04:55:07.458488+0000","last_active":"2024-04-26T04:55:52.499334+0000","last_peered":"2024-04-26T04:55:52.499334+0000","last_clean":"2024-04-26T04:55:52.499334+0000","last_became_active":"2024-04-26T04:55:07.458201+0000","last_became_peered":"2024-04-26T04:55:07.458201+0000","last_unstale":"2024-04-26T04:55:52.499334+0000","last_undegraded":"2024-04-26T04:55:52.499334+0000","last_fullsized":"2024-04-26T04:55:52.499334+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:43:41.715413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.820403+0000","last_change":"2024-04-26T04:55:08.568873+0000","last_active":"2024-04-26T04:55:52.820403+0000","last_peered":"2024-04-26T04:55:52.820403+0000","last_clean":"2024-04-26T04:55:52.820403+0000","last_became_active":"2024-04-26T04:55:08.568566+0000","last_became_peered":"2024-04-26T04:55:08.568566+0000","last_unstale":"2024-04-26T04:55:52.820403+0000","last_undegraded":"2024-04-26T04:55:52.820403+0000","last_fullsized":"2024-04-26T04:55:52.820403+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:51:27.194663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.496231+0000","last_change":"2024-04-26T04:55:07.448595+0000","last_active":"2024-04-26T04:55:52.496231+0000","last_peered":"2024-04-26T04:55:52.496231+0000","last_clean":"2024-04-26T04:55:52.496231+0000","last_became_active":"2024-04-26T04:55:07.448321+0000","last_became_peered":"2024-04-26T04:55:07.448321+0000","last_unstale":"2024-04-26T04:55:52.496231+0000","last_undegraded":"2024-04-26T04:55:52.496231+0000","last_fullsized":"2024-04-26T04:55:52.496231+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:10:46.543937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.499829+0000","last_change":"2024-04-26T04:55:07.502274+0000","last_active":"2024-04-26T04:55:52.499829+0000","last_peered":"2024-04-26T04:55:52.499829+0000","last_clean":"2024-04-26T04:55:52.499829+0000","last_became_active":"2024-04-26T04:55:07.501989+0000","last_became_peered":"2024-04-26T04:55:07.501989+0000","last_unstale":"2024-04-26T04:55:52.499829+0000","last_undegraded":"2024-04-26T04:55:52.499829+0000","last_fullsized":"2024-04-26T04:55:52.499829+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:07:10.637739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.501793+0000","last_change":"2024-04-26T04:55:07.451952+0000","last_active":"2024-04-26T04:55:52.501793+0000","last_peered":"2024-04-26T04:55:52.501793+0000","last_clean":"2024-04-26T04:55:52.501793+0000","last_became_active":"2024-04-26T04:55:07.451660+0000","last_became_peered":"2024-04-26T04:55:07.451660+0000","last_unstale":"2024-04-26T04:55:52.501793+0000","last_undegraded":"2024-04-26T04:55:52.501793+0000","last_fullsized":"2024-04-26T04:55:52.501793+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:36:14.683401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498146+0000","last_change":"2024-04-26T04:55:07.501239+0000","last_active":"2024-04-26T04:55:52.498146+0000","last_peered":"2024-04-26T04:55:52.498146+0000","last_clean":"2024-04-26T04:55:52.498146+0000","last_became_active":"2024-04-26T04:55:07.500968+0000","last_became_peered":"2024-04-26T04:55:07.500968+0000","last_unstale":"2024-04-26T04:55:52.498146+0000","last_undegraded":"2024-04-26T04:55:52.498146+0000","last_fullsized":"2024-04-26T04:55:52.498146+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:39:01.835409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.501758+0000","last_change":"2024-04-26T04:55:07.453480+0000","last_active":"2024-04-26T04:55:52.501758+0000","last_peered":"2024-04-26T04:55:52.501758+0000","last_clean":"2024-04-26T04:55:52.501758+0000","last_became_active":"2024-04-26T04:55:07.453141+0000","last_became_peered":"2024-04-26T04:55:07.453141+0000","last_unstale":"2024-04-26T04:55:52.501758+0000","last_undegraded":"2024-04-26T04:55:52.501758+0000","last_fullsized":"2024-04-26T04:55:52.501758+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:33:34.384745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.496079+0000","last_change":"2024-04-26T04:55:07.500711+0000","last_active":"2024-04-26T04:55:52.496079+0000","last_peered":"2024-04-26T04:55:52.496079+0000","last_clean":"2024-04-26T04:55:52.496079+0000","last_became_active":"2024-04-26T04:55:07.500428+0000","last_became_peered":"2024-04-26T04:55:07.500428+0000","last_unstale":"2024-04-26T04:55:52.496079+0000","last_undegraded":"2024-04-26T04:55:52.496079+0000","last_fullsized":"2024-04-26T04:55:52.496079+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:07:04.979529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.502029+0000","last_change":"2024-04-26T04:55:07.463656+0000","last_active":"2024-04-26T04:55:52.502029+0000","last_peered":"2024-04-26T04:55:52.502029+0000","last_clean":"2024-04-26T04:55:52.502029+0000","last_became_active":"2024-04-26T04:55:07.463417+0000","last_became_peered":"2024-04-26T04:55:07.463417+0000","last_unstale":"2024-04-26T04:55:52.502029+0000","last_undegraded":"2024-04-26T04:55:52.502029+0000","last_fullsized":"2024-04-26T04:55:52.502029+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:56:02.866229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-26T04:55:53.687811+0000","last_change":"2024-04-26T04:55:53.687811+0000","last_active":"2024-04-26T04:55:53.687811+0000","last_peered":"2024-04-26T04:55:53.687811+0000","last_clean":"2024-04-26T04:55:53.687811+0000","last_became_active":"2024-04-26T04:55:53.687402+0000","last_became_peered":"2024-04-26T04:55:53.687402+0000","last_unstale":"2024-04-26T04:55:53.687811+0000","last_undegraded":"2024-04-26T04:55:53.687811+0000","last_fullsized":"2024-04-26T04:55:53.687811+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:04:02.155272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":22,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498864+0000","last_change":"2024-04-26T04:55:07.455662+0000","last_active":"2024-04-26T04:55:52.498864+0000","last_peered":"2024-04-26T04:55:52.498864+0000","last_clean":"2024-04-26T04:55:52.498864+0000","last_became_active":"2024-04-26T04:55:07.455401+0000","last_became_peered":"2024-04-26T04:55:07.455401+0000","last_unstale":"2024-04-26T04:55:52.498864+0000","last_undegraded":"2024-04-26T04:55:52.498864+0000","last_fullsized":"2024-04-26T04:55:52.498864+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:41:47.842841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.820372+0000","last_change":"2024-04-26T04:55:08.602361+0000","last_active":"2024-04-26T04:55:52.820372+0000","last_peered":"2024-04-26T04:55:52.820372+0000","last_clean":"2024-04-26T04:55:52.820372+0000","last_became_active":"2024-04-26T04:55:08.602094+0000","last_became_peered":"2024-04-26T04:55:08.602094+0000","last_unstale":"2024-04-26T04:55:52.820372+0000","last_undegraded":"2024-04-26T04:55:52.820372+0000","last_fullsized":"2024-04-26T04:55:52.820372+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:07:17.911220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.501367+0000","last_change":"2024-04-26T04:55:07.452729+0000","last_active":"2024-04-26T04:55:52.501367+0000","last_peered":"2024-04-26T04:55:52.501367+0000","last_clean":"2024-04-26T04:55:52.501367+0000","last_became_active":"2024-04-26T04:55:07.452426+0000","last_became_peered":"2024-04-26T04:55:07.452426+0000","last_unstale":"2024-04-26T04:55:52.501367+0000","last_undegraded":"2024-04-26T04:55:52.501367+0000","last_fullsized":"2024-04-26T04:55:52.501367+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:59:48.509688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.496372+0000","last_change":"2024-04-26T04:55:07.501192+0000","last_active":"2024-04-26T04:55:52.496372+0000","last_peered":"2024-04-26T04:55:52.496372+0000","last_clean":"2024-04-26T04:55:52.496372+0000","last_became_active":"2024-04-26T04:55:07.500547+0000","last_became_peered":"2024-04-26T04:55:07.500547+0000","last_unstale":"2024-04-26T04:55:52.496372+0000","last_undegraded":"2024-04-26T04:55:52.496372+0000","last_fullsized":"2024-04-26T04:55:52.496372+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:02:18.503080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.820415+0000","last_change":"2024-04-26T04:55:07.462667+0000","last_active":"2024-04-26T04:55:52.820415+0000","last_peered":"2024-04-26T04:55:52.820415+0000","last_clean":"2024-04-26T04:55:52.820415+0000","last_became_active":"2024-04-26T04:55:07.462424+0000","last_became_peered":"2024-04-26T04:55:07.462424+0000","last_unstale":"2024-04-26T04:55:52.820415+0000","last_undegraded":"2024-04-26T04:55:52.820415+0000","last_fullsized":"2024-04-26T04:55:52.820415+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:25:20.792932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.499455+0000","last_change":"2024-04-26T04:55:08.602806+0000","last_active":"2024-04-26T04:55:52.499455+0000","last_peered":"2024-04-26T04:55:52.499455+0000","last_clean":"2024-04-26T04:55:52.499455+0000","last_became_active":"2024-04-26T04:55:08.602515+0000","last_became_peered":"2024-04-26T04:55:08.602515+0000","last_unstale":"2024-04-26T04:55:52.499455+0000","last_undegraded":"2024-04-26T04:55:52.499455+0000","last_fullsized":"2024-04-26T04:55:52.499455+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:45:27.150544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.501675+0000","last_change":"2024-04-26T04:55:07.452078+0000","last_active":"2024-04-26T04:55:52.501675+0000","last_peered":"2024-04-26T04:55:52.501675+0000","last_clean":"2024-04-26T04:55:52.501675+0000","last_became_active":"2024-04-26T04:55:07.451798+0000","last_became_peered":"2024-04-26T04:55:07.451798+0000","last_unstale":"2024-04-26T04:55:52.501675+0000","last_undegraded":"2024-04-26T04:55:52.501675+0000","last_fullsized":"2024-04-26T04:55:52.501675+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:36:16.277372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498265+0000","last_change":"2024-04-26T04:55:07.503315+0000","last_active":"2024-04-26T04:55:52.498265+0000","last_peered":"2024-04-26T04:55:52.498265+0000","last_clean":"2024-04-26T04:55:52.498265+0000","last_became_active":"2024-04-26T04:55:07.502836+0000","last_became_peered":"2024-04-26T04:55:07.502836+0000","last_unstale":"2024-04-26T04:55:52.498265+0000","last_undegraded":"2024-04-26T04:55:52.498265+0000","last_fullsized":"2024-04-26T04:55:52.498265+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:28:24.192527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"69'6","reported_seq":20,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498407+0000","last_change":"2024-04-26T04:55:07.451574+0000","last_active":"2024-04-26T04:55:52.498407+0000","last_peered":"2024-04-26T04:55:52.498407+0000","last_clean":"2024-04-26T04:55:52.498407+0000","last_became_active":"2024-04-26T04:55:07.451154+0000","last_became_peered":"2024-04-26T04:55:07.451154+0000","last_unstale":"2024-04-26T04:55:52.498407+0000","last_undegraded":"2024-04-26T04:55:52.498407+0000","last_fullsized":"2024-04-26T04:55:52.498407+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:56:04.497071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":10,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-26T04:55:53.688486+0000","last_change":"2024-04-26T04:55:53.688486+0000","last_active":"2024-04-26T04:55:53.688486+0000","last_peered":"2024-04-26T04:55:53.688486+0000","last_clean":"2024-04-26T04:55:53.688486+0000","last_became_active":"2024-04-26T04:55:53.687991+0000","last_became_peered":"2024-04-26T04:55:53.687991+0000","last_unstale":"2024-04-26T04:55:53.688486+0000","last_undegraded":"2024-04-26T04:55:53.688486+0000","last_fullsized":"2024-04-26T04:55:53.688486+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:19:30.036450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.501685+0000","last_change":"2024-04-26T04:55:07.454011+0000","last_active":"2024-04-26T04:55:52.501685+0000","last_peered":"2024-04-26T04:55:52.501685+0000","last_clean":"2024-04-26T04:55:52.501685+0000","last_became_active":"2024-04-26T04:55:07.453695+0000","last_became_peered":"2024-04-26T04:55:07.453695+0000","last_unstale":"2024-04-26T04:55:52.501685+0000","last_undegraded":"2024-04-26T04:55:52.501685+0000","last_fullsized":"2024-04-26T04:55:52.501685+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:58:20.724910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.499072+0000","last_change":"2024-04-26T04:55:08.601188+0000","last_active":"2024-04-26T04:55:52.499072+0000","last_peered":"2024-04-26T04:55:52.499072+0000","last_clean":"2024-04-26T04:55:52.499072+0000","last_became_active":"2024-04-26T04:55:08.600844+0000","last_became_peered":"2024-04-26T04:55:08.600844+0000","last_unstale":"2024-04-26T04:55:52.499072+0000","last_undegraded":"2024-04-26T04:55:52.499072+0000","last_fullsized":"2024-04-26T04:55:52.499072+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:16:31.788265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.496409+0000","last_change":"2024-04-26T04:55:07.447684+0000","last_active":"2024-04-26T04:55:52.496409+0000","last_peered":"2024-04-26T04:55:52.496409+0000","last_clean":"2024-04-26T04:55:52.496409+0000","last_became_active":"2024-04-26T04:55:07.447380+0000","last_became_peered":"2024-04-26T04:55:07.447380+0000","last_unstale":"2024-04-26T04:55:52.496409+0000","last_undegraded":"2024-04-26T04:55:52.496409+0000","last_fullsized":"2024-04-26T04:55:52.496409+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:21:18.903588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498856+0000","last_change":"2024-04-26T04:55:08.601318+0000","last_active":"2024-04-26T04:55:52.498856+0000","last_peered":"2024-04-26T04:55:52.498856+0000","last_clean":"2024-04-26T04:55:52.498856+0000","last_became_active":"2024-04-26T04:55:08.601034+0000","last_became_peered":"2024-04-26T04:55:08.601034+0000","last_unstale":"2024-04-26T04:55:52.498856+0000","last_undegraded":"2024-04-26T04:55:52.498856+0000","last_fullsized":"2024-04-26T04:55:52.498856+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:08:22.624996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.820585+0000","last_change":"2024-04-26T04:55:07.457337+0000","last_active":"2024-04-26T04:55:52.820585+0000","last_peered":"2024-04-26T04:55:52.820585+0000","last_clean":"2024-04-26T04:55:52.820585+0000","last_became_active":"2024-04-26T04:55:07.456996+0000","last_became_peered":"2024-04-26T04:55:07.456996+0000","last_unstale":"2024-04-26T04:55:52.820585+0000","last_undegraded":"2024-04-26T04:55:52.820585+0000","last_fullsized":"2024-04-26T04:55:52.820585+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:26:34.645847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498905+0000","last_change":"2024-04-26T04:55:07.503910+0000","last_active":"2024-04-26T04:55:52.498905+0000","last_peered":"2024-04-26T04:55:52.498905+0000","last_clean":"2024-04-26T04:55:52.498905+0000","last_became_active":"2024-04-26T04:55:07.503589+0000","last_became_peered":"2024-04-26T04:55:07.503589+0000","last_unstale":"2024-04-26T04:55:52.498905+0000","last_undegraded":"2024-04-26T04:55:52.498905+0000","last_fullsized":"2024-04-26T04:55:52.498905+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:44:38.391157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498945+0000","last_change":"2024-04-26T04:55:07.461775+0000","last_active":"2024-04-26T04:55:52.498945+0000","last_peered":"2024-04-26T04:55:52.498945+0000","last_clean":"2024-04-26T04:55:52.498945+0000","last_became_active":"2024-04-26T04:55:07.461493+0000","last_became_peered":"2024-04-26T04:55:07.461493+0000","last_unstale":"2024-04-26T04:55:52.498945+0000","last_undegraded":"2024-04-26T04:55:52.498945+0000","last_fullsized":"2024-04-26T04:55:52.498945+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:00:46.837734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.820396+0000","last_change":"2024-04-26T04:55:08.601565+0000","last_active":"2024-04-26T04:55:52.820396+0000","last_peered":"2024-04-26T04:55:52.820396+0000","last_clean":"2024-04-26T04:55:52.820396+0000","last_became_active":"2024-04-26T04:55:08.601226+0000","last_became_peered":"2024-04-26T04:55:08.601226+0000","last_unstale":"2024-04-26T04:55:52.820396+0000","last_undegraded":"2024-04-26T04:55:52.820396+0000","last_fullsized":"2024-04-26T04:55:52.820396+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:15:30.591873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.495323+0000","last_change":"2024-04-26T04:55:07.457162+0000","last_active":"2024-04-26T04:55:52.495323+0000","last_peered":"2024-04-26T04:55:52.495323+0000","last_clean":"2024-04-26T04:55:52.495323+0000","last_became_active":"2024-04-26T04:55:07.456801+0000","last_became_peered":"2024-04-26T04:55:07.456801+0000","last_unstale":"2024-04-26T04:55:52.495323+0000","last_undegraded":"2024-04-26T04:55:52.495323+0000","last_fullsized":"2024-04-26T04:55:52.495323+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:39:42.692710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.499622+0000","last_change":"2024-04-26T04:55:08.569011+0000","last_active":"2024-04-26T04:55:52.499622+0000","last_peered":"2024-04-26T04:55:52.499622+0000","last_clean":"2024-04-26T04:55:52.499622+0000","last_became_active":"2024-04-26T04:55:08.568533+0000","last_became_peered":"2024-04-26T04:55:08.568533+0000","last_unstale":"2024-04-26T04:55:52.499622+0000","last_undegraded":"2024-04-26T04:55:52.499622+0000","last_fullsized":"2024-04-26T04:55:52.499622+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:53:08.464353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:55:52.498275+0000","last_change":"2024-04-26T04:55:07.451596+0000","last_active":"2024-04-26T04:55:52.498275+0000","last_peered":"2024-04-26T04:55:52.498275+0000","last_clean":"2024-04-26T04:55:52.498275+0000","last_became_active":"2024-04-26T04:55:07.451265+0000","last_became_peered":"2024-04-26T04:55:07.451265+0000","last_unstale":"2024-04-26T04:55:52.498275+0000","last_undegraded":"2024-04-26T04:55:52.498275+0000","last_fullsized":"2024-04-26T04:55:52.498275+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:26:45.706484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201316,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298672,"kb_used_data":1312,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446480,"statfs":{"total":95995035648,"available":95689195520,"internally_reserved":0,"allocated":1343488,"data_stored":908282,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:55:26 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.54200000000000004,"15min":0.54400000000000004},"min":{"1min":0.33000000000000002,"5min":0.315,"15min":0.315},"max":{"1min":0.67700000000000005,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.503,"5min":0.51900000000000002,"15min":0.52100000000000002},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.626}]},{"osd":1,"last update":"Fri Apr 26 04:55:26 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53900000000000003,"15min":0.54000000000000004},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.40400000000000003}]},{"osd":2,"last update":"Fri Apr 26 04:55:26 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55900000000000005,"15min":0.56000000000000005},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.45800000000000002}]},{"osd":3,"last update":"Fri Apr 26 04:55:26 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.55100000000000005}]},{"osd":4,"last update":"Fri Apr 26 04:55:26 2024","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.45300000000000001,"15min":0.44900000000000001},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.495,"15min":0.49099999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.56799999999999995}]},{"osd":5,"last update":"Fri Apr 26 04:55:26 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.48499999999999999,"15min":0.47799999999999998},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.51300000000000001,"15min":0.50700000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.58699999999999997}]},{"osd":6,"last update":"Fri Apr 26 04:55:26 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.53000000000000003,"15min":0.52600000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.69899999999999995,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.57199999999999995,"15min":0.56799999999999995},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.76200000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.61199999999999999}]}]},{"osd":6,"up_from":47,"seq":201863462953,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298652,"kb_used_data":1296,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446500,"statfs":{"total":95995035648,"available":95689216000,"internally_reserved":0,"allocated":1327104,"data_stored":907921,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:56:00 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.53600000000000003,"15min":0.53400000000000003},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.628},{"interface":"front","average":{"1min":0.755,"5min":0.57999999999999996,"15min":0.55000000000000004},"min":{"1min":0.39800000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":4.6219999999999999,"5min":4.6219999999999999,"15min":4.6219999999999999},"last":0.63800000000000001}]},{"osd":1,"last update":"Fri Apr 26 04:56:00 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.76400000000000001,"15min":0.60399999999999998},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.745,"5min":18.579000000000001,"15min":18.579000000000001},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.52300000000000002,"15min":0.51400000000000001},"min":{"1min":0.42899999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.59499999999999997}]},{"osd":2,"last update":"Fri Apr 26 04:56:00 2024","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":0.74299999999999999,"15min":0.77300000000000002},"min":{"1min":0.39800000000000002,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":4.5789999999999997,"5min":5.6980000000000004,"15min":5.6980000000000004},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.434,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.73299999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.53600000000000003}]},{"osd":3,"last update":"Fri Apr 26 04:56:00 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54500000000000004},"min":{"1min":0.42899999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.71699999999999997,"15min":0.77300000000000002},"min":{"1min":0.48099999999999998,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.72199999999999998,"5min":5.6369999999999996,"15min":5.6369999999999996},"last":0.55900000000000005}]},{"osd":4,"last update":"Fri Apr 26 04:56:00 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.442,"15min":0.42499999999999999},"min":{"1min":0.29399999999999998,"5min":0.255,"15min":0.255},"max":{"1min":0.66800000000000004,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.46100000000000002,"15min":0.44},"min":{"1min":0.311,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.51200000000000001}]},{"osd":5,"last update":"Fri Apr 26 04:56:00 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.45700000000000002,"15min":0.435},"min":{"1min":0.33200000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.65000000000000002,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.47599999999999998,"15min":0.45200000000000001},"min":{"1min":0.38500000000000001,"5min":0.308,"15min":0.308},"max":{"1min":0.73899999999999999,"5min":0.753,"15min":0.753},"last":0.65100000000000002}]},{"osd":7,"last update":"Fri Apr 26 04:55:25 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.48999999999999999,"15min":0.48599999999999999},"min":{"1min":0.377,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.504,"5min":0.501,"15min":0.501},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.76500000000000001,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.48399999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574913,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298068,"kb_used_data":708,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447084,"statfs":{"total":95995035648,"available":95689814016,"internally_reserved":0,"allocated":724992,"data_stored":316469,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:55:09 2024","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.36199999999999999,"15min":0.32700000000000001},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.373,"15min":0.33500000000000002},"min":{"1min":0.27600000000000002,"5min":0.214,"15min":0.214},"max":{"1min":0.62,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.379}]},{"osd":2,"last update":"Fri Apr 26 04:55:29 2024","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.40000000000000002,"15min":0.38800000000000001},"min":{"1min":0.29099999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.42199999999999999,"15min":0.41599999999999998},"min":{"1min":0.28999999999999998,"5min":0.22,"15min":0.22},"max":{"1min":0.62,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.42899999999999999}]},{"osd":3,"last update":"Fri Apr 26 04:55:58 2024","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.40899999999999997,"15min":0.41099999999999998},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.52700000000000002,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.46899999999999997,"15min":0.47899999999999998},"min":{"1min":0.32100000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.60699999999999998,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.40200000000000002}]},{"osd":4,"last update":"Fri Apr 26 04:55:16 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.56200000000000006,"15min":0.54700000000000004},"min":{"1min":0.41399999999999998,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.77700000000000002,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.55000000000000004,"15min":0.53400000000000003},"min":{"1min":0.42799999999999999,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.79300000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.52300000000000002}]},{"osd":5,"last update":"Fri Apr 26 04:55:45 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.60299999999999998,"15min":0.59899999999999998},"min":{"1min":0.41499999999999998,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.82399999999999995,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.60799999999999998,"15min":0.60899999999999999},"min":{"1min":0.44,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.77300000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.57599999999999996}]},{"osd":6,"last update":"Fri Apr 26 04:56:07 2024","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.64200000000000002,"15min":0.65000000000000002},"min":{"1min":0.46400000000000002,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.90900000000000003,"5min":1.0269999999999999,"15min":1.0269999999999999},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.63100000000000001,"5min":1.399,"15min":1.7370000000000001},"min":{"1min":0.45000000000000001,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.873,"5min":24.962,"15min":24.962},"last":0.56399999999999995}]},{"osd":7,"last update":"Fri Apr 26 04:55:25 2024","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.84599999999999997,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.65500000000000003,"15min":0.65700000000000003},"min":{"1min":0.498,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.81799999999999995,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.61699999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738437,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298652,"kb_used_data":1292,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446500,"statfs":{"total":95995035648,"available":95689216000,"internally_reserved":0,"allocated":1323008,"data_stored":907164,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 26 04:55:10 2024","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.39100000000000001,"15min":0.36799999999999999},"min":{"1min":0.25800000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.34999999999999998,"5min":0.39300000000000002,"15min":0.38300000000000001},"min":{"1min":0.26800000000000002,"5min":0.22,"15min":0.22},"max":{"1min":0.51200000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.33200000000000002}]},{"osd":2,"last update":"Fri Apr 26 04:55:39 2024","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.40600000000000003,"15min":0.40400000000000003},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.53500000000000003,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.45100000000000001,"15min":0.45600000000000002},"min":{"1min":0.26500000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.34699999999999998}]},{"osd":3,"last update":"Fri Apr 26 04:55:55 2024","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.46500000000000002,"15min":0.48299999999999998},"min":{"1min":0.33900000000000002,"5min":0.249,"15min":0.249},"max":{"1min":1.1140000000000001,"5min":1.1140000000000001,"15min":1.1140000000000001},"last":0.378},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.497,"15min":0.50800000000000001},"min":{"1min":0.36199999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.0209999999999999,"5min":1.0209999999999999,"15min":1.0209999999999999},"last":0.40699999999999997}]},{"osd":4,"last update":"Fri Apr 26 04:55:17 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.623,"15min":0.63700000000000001},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59999999999999998,"15min":0.60599999999999998},"min":{"1min":0.44800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.82699999999999996,"5min":0.872,"15min":0.872},"last":0.46000000000000002}]},{"osd":5,"last update":"Fri Apr 26 04:55:39 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.60999999999999999,"15min":0.62},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.76200000000000001,"5min":0.88,"15min":0.88},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.626,"15min":0.64200000000000002},"min":{"1min":0.41899999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.79100000000000004,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.59499999999999997}]},{"osd":6,"last update":"Fri Apr 26 04:56:01 2024","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63200000000000001,"15min":0.64000000000000001},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.0669999999999999,"5min":1.0669999999999999,"15min":1.0669999999999999},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.66500000000000004,"15min":0.67500000000000004},"min":{"1min":0.46600000000000003,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.88800000000000001,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.58199999999999996}]},{"osd":7,"last update":"Fri Apr 26 04:55:26 2024","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.65800000000000003,"15min":0.66100000000000003},"min":{"1min":0.48899999999999999,"5min":0.434,"15min":0.434},"max":{"1min":0.95899999999999996,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.625},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.63,"15min":0.63200000000000001},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.77700000000000002,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.55800000000000005}]}]},{"osd":2,"up_from":18,"seq":77309411388,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298072,"kb_used_data":716,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447080,"statfs":{"total":95995035648,"available":95689809920,"internally_reserved":0,"allocated":733184,"data_stored":317073,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:55:31 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.38,"15min":0.38600000000000001},"min":{"1min":0.26400000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.57499999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.27700000000000002},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.45700000000000002,"15min":0.441},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.61399999999999999,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.371}]},{"osd":1,"last update":"Fri Apr 26 04:55:31 2024","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.40799999999999997,"15min":0.41199999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.42599999999999999,"15min":0.42399999999999999},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.53900000000000003,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.39500000000000002}]},{"osd":3,"last update":"Fri Apr 26 04:55:59 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.47099999999999997,"15min":0.48099999999999998},"min":{"1min":0.307,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.64000000000000001,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.49399999999999999,"15min":0.51400000000000001},"min":{"1min":0.33100000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.72099999999999997,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.32000000000000001}]},{"osd":4,"last update":"Fri Apr 26 04:55:16 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.58399999999999996,"15min":0.59499999999999997},"min":{"1min":0.40300000000000002,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.79900000000000004,"5min":1.042,"15min":1.042},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.61599999999999999,"15min":0.628},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.72699999999999998,"5min":1.1240000000000001,"15min":1.1240000000000001},"last":0.73299999999999998}]},{"osd":5,"last update":"Fri Apr 26 04:55:40 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.64200000000000002,"15min":0.66600000000000004},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.80700000000000005,"5min":1.077,"15min":1.077},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.61499999999999999,"15min":0.627},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.89200000000000002,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.76200000000000001}]},{"osd":6,"last update":"Fri Apr 26 04:55:59 2024","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.65000000000000002,"15min":0.66400000000000003},"min":{"1min":0.45100000000000001,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.876,"5min":1.012,"15min":1.012},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.65900000000000003,"15min":0.68100000000000005},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.93000000000000005,"5min":1.0960000000000001,"15min":1.0960000000000001},"last":0.71999999999999997}]},{"osd":7,"last update":"Fri Apr 26 04:55:22 2024","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.65900000000000003,"15min":0.66100000000000003},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.81200000000000006,"5min":0.998,"15min":0.998},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.626,"15min":0.628},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.85699999999999998,"5min":0.875,"15min":0.875},"last":0.79300000000000004}]}]},{"osd":3,"up_from":26,"seq":111669149750,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298072,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447080,"statfs":{"total":95995035648,"available":95689809920,"internally_reserved":0,"allocated":729088,"data_stored":316559,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:55:57 2024","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.42799999999999999,"15min":0.45500000000000002},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.48199999999999998,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.42099999999999999,"15min":0.44900000000000001},"min":{"1min":0.26300000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.39400000000000002}]},{"osd":1,"last update":"Fri Apr 26 04:55:57 2024","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.45300000000000001,"15min":0.48299999999999998},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.56899999999999995,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.44400000000000001,"15min":0.47499999999999998},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.59299999999999997,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.34799999999999998}]},{"osd":2,"last update":"Fri Apr 26 04:55:57 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.48199999999999998,"15min":0.499},"min":{"1min":0.34999999999999998,"5min":0.317,"15min":0.317},"max":{"1min":0.57299999999999995,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.45400000000000001,"15min":0.498},"min":{"1min":0.29199999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.55300000000000005,"5min":0.748,"15min":0.748},"last":0.317}]},{"osd":4,"last update":"Fri Apr 26 04:55:17 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.61099999999999999,"15min":0.61799999999999999},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59299999999999997,"15min":0.60099999999999998},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.73299999999999998}]},{"osd":5,"last update":"Fri Apr 26 04:55:41 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":1.1160000000000001,"15min":1.371},"min":{"1min":0.41299999999999998,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.86299999999999999,"5min":18.239000000000001,"15min":18.239000000000001},"last":0.62},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.65000000000000002,"15min":0.65800000000000003},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.82899999999999996,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.69399999999999995}]},{"osd":6,"last update":"Fri Apr 26 04:56:00 2024","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":1.2350000000000001,"15min":1.5029999999999999},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.94299999999999995,"5min":18.221,"15min":18.221},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.70399999999999996}]},{"osd":7,"last update":"Fri Apr 26 04:55:26 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.56399999999999995,"15min":0.55600000000000005},"min":{"1min":0.45700000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.69699999999999995,"5min":0.67600000000000005,"15min":0.67200000000000004},"min":{"1min":0.53600000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.67000000000000004}]}]},{"osd":4,"up_from":31,"seq":133143986226,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298092,"kb_used_data":732,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447060,"statfs":{"total":95995035648,"available":95689789440,"internally_reserved":0,"allocated":749568,"data_stored":318167,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:55:15 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.502,"15min":0.49399999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.746,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.52500000000000002,"15min":0.52200000000000002},"min":{"1min":0.316,"5min":0.309,"15min":0.309},"max":{"1min":0.80900000000000005,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.44700000000000001}]},{"osd":1,"last update":"Fri Apr 26 04:55:15 2024","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":0.55300000000000005,"15min":0.51500000000000001},"min":{"1min":0.33900000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":4.3700000000000001,"5min":4.3700000000000001,"15min":4.3700000000000001},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.51700000000000002,"15min":0.51200000000000001},"min":{"1min":0.34399999999999997,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.39900000000000002}]},{"osd":2,"last update":"Fri Apr 26 04:55:15 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.52400000000000002,"15min":0.52900000000000003},"min":{"1min":0.372,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.496,"5min":0.50600000000000001,"15min":0.501},"min":{"1min":0.36299999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.65800000000000003,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.496}]},{"osd":3,"last update":"Fri Apr 26 04:55:15 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.54300000000000004,"15min":0.53900000000000003},"min":{"1min":0.41299999999999998,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.71599999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.54900000000000004,"15min":0.53900000000000003},"min":{"1min":0.42599999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.78700000000000003,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.66600000000000004}]},{"osd":5,"last update":"Fri Apr 26 04:55:46 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.41299999999999998,"15min":0.39100000000000001},"min":{"1min":0.33100000000000002,"5min":0.221,"15min":0.221},"max":{"1min":0.61099999999999999,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.42099999999999999,"15min":0.39800000000000002},"min":{"1min":0.34699999999999998,"5min":0.251,"15min":0.251},"max":{"1min":0.54600000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.55900000000000005}]},{"osd":6,"last update":"Fri Apr 26 04:56:07 2024","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.46400000000000002,"15min":0.46000000000000002},"min":{"1min":0.34300000000000003,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.59399999999999997,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.51600000000000001,"15min":0.51700000000000002},"min":{"1min":0.38300000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.71499999999999997,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.52300000000000002}]},{"osd":7,"last update":"Fri Apr 26 04:55:25 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.52500000000000002,"15min":0.52200000000000002},"min":{"1min":0.432,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.69999999999999996,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.52300000000000002,"15min":0.52100000000000002},"min":{"1min":0.40200000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.67100000000000004,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.61399999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724590,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298068,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447084,"statfs":{"total":95995035648,"available":95689814016,"internally_reserved":0,"allocated":729088,"data_stored":316786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:55:37 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.53500000000000003,"15min":0.54800000000000004},"min":{"1min":0.32600000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.70299999999999996,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.55100000000000005,"15min":0.55700000000000005},"min":{"1min":0.41399999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.68200000000000005,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.63800000000000001}]},{"osd":1,"last update":"Fri Apr 26 04:55:37 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.53900000000000003,"15min":0.54900000000000004},"min":{"1min":0.40300000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.627,"5min":0.75,"15min":0.75},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.55200000000000005,"15min":0.56299999999999994},"min":{"1min":0.373,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.73499999999999999,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.68100000000000005}]},{"osd":2,"last update":"Fri Apr 26 04:55:37 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55600000000000005,"15min":0.56399999999999995},"min":{"1min":0.40600000000000003,"5min":0.317,"15min":0.317},"max":{"1min":0.72299999999999998,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.57199999999999995,"15min":0.58999999999999997},"min":{"1min":0.36699999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.69299999999999995,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.60199999999999998}]},{"osd":3,"last update":"Fri Apr 26 04:55:37 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.60399999999999998,"15min":0.62},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.72499999999999998,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.75900000000000001},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.59699999999999998,"15min":0.60699999999999998},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.69099999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.69799999999999995}]},{"osd":4,"last update":"Fri Apr 26 04:55:37 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.41299999999999998,"15min":0.39200000000000002},"min":{"1min":0.32000000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.433,"15min":0.40500000000000003},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.78100000000000003,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.61699999999999999}]},{"osd":6,"last update":"Fri Apr 26 04:56:04 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.46100000000000002,"15min":0.443},"min":{"1min":0.35399999999999998,"5min":0.31,"15min":0.31},"max":{"1min":0.76300000000000001,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.73999999999999999},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.51900000000000002,"15min":0.499},"min":{"1min":0.34999999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.81299999999999994,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.78100000000000003}]},{"osd":7,"last update":"Fri Apr 26 04:55:25 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.47499999999999998,"15min":0.46899999999999997},"min":{"1min":0.33700000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.751,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.51400000000000001,"15min":0.51100000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.57399999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-26T04:56:10.174 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-26T04:56:10.175 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-26T04:56:10.175 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-26T04:56:10.175 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:56:10.442 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:10.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:10 smithi116 ceph-mon[29897]: from='client.24692 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:56:10.845 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:10 smithi012 ceph-mon[26617]: from='client.24692 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:56:10.845 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:10 smithi012 ceph-mon[37567]: from='client.24692 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:56:11.700 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:11 smithi012 ceph-mon[26617]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:11.700 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:11 smithi012 ceph-mon[37567]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:11.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:11 smithi116 ceph-mon[29897]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:12.176 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T04:56:12.177 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-26T04:56:12.779 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-26T04:56:12.779 INFO:teuthology.run_tasks:Running task print... 2024-04-26T04:56:12.792 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2024-04-26T04:56:12.792 INFO:teuthology.run_tasks:Running task parallel... 2024-04-26T04:56:12.833 INFO:teuthology.task.parallel:starting parallel... 2024-04-26T04:56:12.833 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-26T04:56:12.834 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-04-26T04:56:12.837 INFO:tasks.workunit:Pulling workunits from ref quincy 2024-04-26T04:56:12.838 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-04-26T04:56:12.838 DEBUG:teuthology.orchestra.run.smithi012:> stat -- /home/ubuntu/cephtest/mnt.0 2024-04-26T04:56:12.838 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-26T04:56:12.838 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-04-26T04:56:12.839 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi012.front.sepia.ceph.com 2024-04-26T04:56:12.839 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2024-04-26T04:56:12.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T04:56:12.857 INFO:teuthology.orchestra.run.smithi012.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-04-26T04:56:12.857 DEBUG:teuthology.orchestra.run.smithi012:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-04-26T04:56:12.864 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:12 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/2024902493' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:56:12.864 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:12 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/2024902493' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:56:12.865 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:56:12 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:56:12] "GET /metrics HTTP/1.1" 200 239110 "" "Prometheus/2.43.0" 2024-04-26T04:56:12.880 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-04-26T04:56:12.880 DEBUG:teuthology.orchestra.run.smithi012:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-04-26T04:56:12.937 INFO:tasks.workunit:timeout=3h 2024-04-26T04:56:12.937 INFO:tasks.workunit:cleanup=True 2024-04-26T04:56:12.937 DEBUG:teuthology.orchestra.run.smithi012:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-04-26T04:56:12.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:12 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/2024902493' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:56:12.992 INFO:tasks.workunit.client.0.smithi012.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-04-26T04:56:13.109 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:13.909 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:13 smithi012 ceph-mon[26617]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:13.910 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:13 smithi012 ceph-mon[37567]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:13.991 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:56:13 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:56:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:56:13.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:13 smithi116 ceph-mon[29897]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:15.466 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:56:15 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: [26/Apr/2024:04:56:15] ENGINE Bus STOPPING 2024-04-26T04:56:15.467 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:56:15 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: [26/Apr/2024:04:56:15] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:56:15.467 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:56:15 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: [26/Apr/2024:04:56:15] ENGINE Bus STOPPED 2024-04-26T04:56:15.467 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:56:15 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: [26/Apr/2024:04:56:15] ENGINE Bus STARTING 2024-04-26T04:56:15.467 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:56:15 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: [26/Apr/2024:04:56:15] ENGINE Serving on http://:::9283 2024-04-26T04:56:15.467 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:56:15 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: [26/Apr/2024:04:56:15] ENGINE Bus STARTED 2024-04-26T04:56:15.762 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:15 smithi012 ceph-mon[37567]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:15.762 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:15 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/930362659' entity='client.admin' 2024-04-26T04:56:15.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:15 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:56:15.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:15 smithi012 ceph-mon[26617]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:15.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:15 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/930362659' entity='client.admin' 2024-04-26T04:56:15.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:15 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:56:15.827 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph config set global log_to_journald false --force' 2024-04-26T04:56:15.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:15 smithi116 ceph-mon[29897]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:15.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:15 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/930362659' entity='client.admin' 2024-04-26T04:56:15.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:15 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:56:16.149 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:16.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:16 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:56:16.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:16 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:56:16.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:16 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:56:17.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:16 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:56:17.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:16 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:56:17.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:16 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:56:17.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:16 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:56:17.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:16 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:56:17.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:16 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:56:17.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:17 smithi012 ceph-mon[26617]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:17.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:17 smithi012 ceph-mon[37567]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:17.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:17 smithi116 ceph-mon[29897]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:18.561 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-26T04:56:18.827 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:19.236 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:18 smithi012 ceph-mon[26617]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:19.236 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:18 smithi012 ceph-mon[37567]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:19.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:18 smithi116 ceph-mon[29897]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:20.638 INFO:teuthology.orchestra.run.smithi012.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:56:21.377 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-26T04:56:21.649 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:21.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:21 smithi012 ceph-mon[26617]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:21.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:21 smithi012 ceph-mon[26617]: from='client.24716 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:56:21.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:21 smithi012 ceph-mon[26617]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:56:21.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:21 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:56:21.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:21 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:56:21.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:21 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:56:21.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:21 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:56:21.927 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:21 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:56:21.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:21 smithi012 ceph-mon[37567]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:21.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:21 smithi012 ceph-mon[37567]: from='client.24716 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:56:21.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:21 smithi012 ceph-mon[37567]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:56:21.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:21 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:56:21.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:21 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:56:21.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:21 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:56:21.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:21 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:56:21.928 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:21 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:56:21.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:21 smithi116 ceph-mon[29897]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:21.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:21 smithi116 ceph-mon[29897]: from='client.24716 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:56:21.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:21 smithi116 ceph-mon[29897]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:56:21.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:21 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:56:21.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:21 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:56:21.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:21 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:56:21.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:21 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:56:21.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:21 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:56:22.933 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:22 smithi012 ceph-mon[26617]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:56:22.933 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:22 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:22.934 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:56:22 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:56:22] "GET /metrics HTTP/1.1" 200 239100 "" "Prometheus/2.43.0" 2024-04-26T04:56:22.934 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:22 smithi012 ceph-mon[37567]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:56:22.934 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:22 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:22.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:22 smithi116 ceph-mon[29897]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:56:22.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:22 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:23.757 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T04:56:23.757 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.a smithi012 *:9093,9094 running (80s) 37s ago 2m 16.9M - 0.25.0 c8568f914cd2 10573377212b 2024-04-26T04:56:23.757 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.a smithi116 *:3000 running (2m) 55s ago 2m 84.5M - 9.4.7 954c08fa6188 c7fd7ca82930 2024-04-26T04:56:23.757 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foo.smithi012.fdqwmo smithi012 running (66s) 37s ago 66s 13.5M - 17.2.7-902-gc61472c8 0de407c1b3af 53f6c82f01b4 2024-04-26T04:56:23.757 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foo.smithi116.kduyef smithi116 running (64s) 55s ago 63s 13.3M - 17.2.7-902-gc61472c8 0de407c1b3af 4c0a0b911593 2024-04-26T04:56:23.757 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.x smithi116 *:8443 running (6m) 55s ago 6m 431M - 17.2.7-902-gc61472c8 0de407c1b3af dd1f1c263e39 2024-04-26T04:56:23.757 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.y smithi012 *:9283 running (7m) 37s ago 7m 475M - 17.2.7-902-gc61472c8 0de407c1b3af cdd1f8559c4e 2024-04-26T04:56:23.757 INFO:teuthology.orchestra.run.smithi012.stdout:mon.a smithi012 running (8m) 37s ago 8m 42.6M 2048M 17.2.7-902-gc61472c8 0de407c1b3af 1b1568ad49e7 2024-04-26T04:56:23.758 INFO:teuthology.orchestra.run.smithi012.stdout:mon.b smithi116 running (6m) 55s ago 6m 35.3M 2048M 17.2.7-902-gc61472c8 0de407c1b3af e6cd16f01417 2024-04-26T04:56:23.758 INFO:teuthology.orchestra.run.smithi012.stdout:mon.c smithi012 running (6m) 37s ago 6m 35.6M 2048M 17.2.7-902-gc61472c8 0de407c1b3af 0f03d4c16528 2024-04-26T04:56:23.758 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.a smithi012 *:9100 running (107s) 37s ago 98s 13.6M - 1.5.0 0da6a335fe13 991e63fb0076 2024-04-26T04:56:23.758 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.b smithi116 *:9100 running (95s) 55s ago 94s 10.4M - 1.5.0 0da6a335fe13 da87ef4e90ea 2024-04-26T04:56:23.758 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (5m) 37s ago 5m 41.6M 4017M 17.2.7-902-gc61472c8 0de407c1b3af 25b6843697db 2024-04-26T04:56:23.758 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (5m) 37s ago 5m 40.6M 4017M 17.2.7-902-gc61472c8 0de407c1b3af 8117a7da3ae3 2024-04-26T04:56:23.758 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (5m) 37s ago 5m 41.2M 4017M 17.2.7-902-gc61472c8 0de407c1b3af 432af63060b8 2024-04-26T04:56:23.758 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (4m) 37s ago 4m 42.7M 4017M 17.2.7-902-gc61472c8 0de407c1b3af f65d4289a970 2024-04-26T04:56:23.758 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi116 running (4m) 55s ago 4m 39.7M 4016M 17.2.7-902-gc61472c8 0de407c1b3af 6b0bbf77cb39 2024-04-26T04:56:23.758 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi116 running (3m) 55s ago 3m 39.5M 4016M 17.2.7-902-gc61472c8 0de407c1b3af a551ef15fa7e 2024-04-26T04:56:23.758 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi116 running (3m) 55s ago 3m 39.9M 4016M 17.2.7-902-gc61472c8 0de407c1b3af cdc0eb6f4fa2 2024-04-26T04:56:23.758 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi116 running (3m) 55s ago 3m 40.5M 4016M 17.2.7-902-gc61472c8 0de407c1b3af 3f6689213fda 2024-04-26T04:56:23.758 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.a smithi116 *:9095 running (75s) 55s ago 2m 35.7M - 2.43.0 a07b618ecd1d db82def51910 2024-04-26T04:56:23.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:23 smithi116 ceph-mon[29897]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:23.992 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:56:23 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:56:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:56:24.012 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:23 smithi012 ceph-mon[37567]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:24.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:23 smithi012 ceph-mon[26617]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:24.622 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'echo wait for minority of mons to upgrade' 2024-04-26T04:56:24.921 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:25.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:25 smithi116 ceph-mon[29897]: from='client.24722 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:56:25.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:25 smithi116 ceph-mon[29897]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:26.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:25 smithi012 ceph-mon[26617]: from='client.24722 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:56:26.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:25 smithi012 ceph-mon[26617]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:26.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:25 smithi012 ceph-mon[37567]: from='client.24722 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:56:26.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:25 smithi012 ceph-mon[37567]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:26.534 DEBUG:teuthology.orchestra.run.smithi012:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-04-26T04:56:26.555 INFO:tasks.workunit.client.0.smithi012.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-04-26T04:56:26.558 INFO:tasks.workunit.client.0.smithi012.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-04-26T04:56:26.558 INFO:tasks.workunit.client.0.smithi012.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-04-26T04:56:27.246 INFO:teuthology.orchestra.run.smithi012.stdout:wait for minority of mons to upgrade 2024-04-26T04:56:27.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:27 smithi116 ceph-mon[29897]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:28.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:27 smithi012 ceph-mon[26617]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:28.012 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:27 smithi012 ceph-mon[37567]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:28.088 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2024-04-26T04:56:28.191 INFO:tasks.workunit.client.0.smithi012.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-04-26T04:56:28.251 INFO:tasks.workunit.client.0.smithi012.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-04-26T04:56:28.293 INFO:tasks.workunit.client.0.smithi012.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-04-26T04:56:28.294 INFO:tasks.workunit.client.0.smithi012.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-04-26T04:56:28.295 INFO:tasks.workunit.client.0.smithi012.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-04-26T04:56:28.337 INFO:tasks.workunit.client.0.smithi012.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-04-26T04:56:28.340 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-04-26T04:56:28.340 DEBUG:teuthology.orchestra.run.smithi012:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-04-26T04:56:28.357 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2024-04-26T04:56:28.358 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2024-04-26T04:56:28.358 DEBUG:teuthology.orchestra.run.smithi012:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2024-04-26T04:56:28.428 INFO:tasks.workunit.client.0.smithi012.stdout:Running main() from gmock_main.cc 2024-04-26T04:56:28.428 INFO:tasks.workunit.client.0.smithi012.stdout:[==========] Running 70 tests from 1 test suite. 2024-04-26T04:56:28.428 INFO:tasks.workunit.client.0.smithi012.stdout:[----------] Global test environment set-up. 2024-04-26T04:56:28.428 INFO:tasks.workunit.client.0.smithi012.stdout:[----------] 70 tests from TestClsRbd 2024-04-26T04:56:28.472 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:56:29.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:29 smithi012 ceph-mon[26617]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:29.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:29 smithi012 ceph-mon[37567]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:29.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:29 smithi116 ceph-mon[29897]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:30.009 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.get_all_features 2024-04-26T04:56:30.025 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.get_all_features (17 ms) 2024-04-26T04:56:30.026 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.copyup 2024-04-26T04:56:30.219 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.copyup (194 ms) 2024-04-26T04:56:30.219 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.sparse_copyup 2024-04-26T04:56:30.231 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.sparse_copyup (13 ms) 2024-04-26T04:56:30.232 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.get_and_set_id 2024-04-26T04:56:30.239 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.get_and_set_id (8 ms) 2024-04-26T04:56:30.239 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.add_remove_child 2024-04-26T04:56:30.251 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.add_remove_child (11 ms) 2024-04-26T04:56:30.251 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.directory_methods 2024-04-26T04:56:30.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:30 smithi012 ceph-mon[26617]: osdmap e72: 8 total, 8 up, 8 in 2024-04-26T04:56:30.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:30 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/3774147553' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi012-77771-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:56:30.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:30 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/3774147553' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi012-77771-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:56:30.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:30 smithi012 ceph-mon[26617]: osdmap e73: 8 total, 8 up, 8 in 2024-04-26T04:56:30.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:30 smithi012 ceph-mon[37567]: osdmap e72: 8 total, 8 up, 8 in 2024-04-26T04:56:30.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:30 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/3774147553' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi012-77771-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:56:30.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:30 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/3774147553' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi012-77771-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:56:30.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:30 smithi012 ceph-mon[37567]: osdmap e73: 8 total, 8 up, 8 in 2024-04-26T04:56:30.279 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.directory_methods (28 ms) 2024-04-26T04:56:30.279 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.create 2024-04-26T04:56:30.292 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.create (13 ms) 2024-04-26T04:56:30.292 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.get_features 2024-04-26T04:56:30.293 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.get_features (1 ms) 2024-04-26T04:56:30.293 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.get_object_prefix 2024-04-26T04:56:30.295 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.get_object_prefix (2 ms) 2024-04-26T04:56:30.295 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2024-04-26T04:56:30.296 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.get_create_timestamp (2 ms) 2024-04-26T04:56:30.297 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2024-04-26T04:56:30.298 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.get_access_timestamp (1 ms) 2024-04-26T04:56:30.298 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2024-04-26T04:56:30.299 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.get_modify_timestamp (1 ms) 2024-04-26T04:56:30.299 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.get_data_pool 2024-04-26T04:56:30.302 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.get_data_pool (3 ms) 2024-04-26T04:56:30.302 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.get_size 2024-04-26T04:56:30.306 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.get_size (4 ms) 2024-04-26T04:56:30.306 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.set_size 2024-04-26T04:56:30.311 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.set_size (5 ms) 2024-04-26T04:56:30.311 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.protection_status 2024-04-26T04:56:30.333 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.protection_status (22 ms) 2024-04-26T04:56:30.333 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.snapshot_limits 2024-04-26T04:56:30.341 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.snapshot_limits (8 ms) 2024-04-26T04:56:30.341 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.parents_v1 2024-04-26T04:56:30.386 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.parents_v1 (45 ms) 2024-04-26T04:56:30.386 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.parents_v2 2024-04-26T04:56:30.418 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.parents_v2 (29 ms) 2024-04-26T04:56:30.418 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.snapshots 2024-04-26T04:56:30.436 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.snapshots (21 ms) 2024-04-26T04:56:30.436 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.snapid_race 2024-04-26T04:56:30.443 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.snapid_race (7 ms) 2024-04-26T04:56:30.443 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.stripingv2 2024-04-26T04:56:30.451 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.stripingv2 (9 ms) 2024-04-26T04:56:30.452 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.object_map_save 2024-04-26T04:56:30.454 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.object_map_save (2 ms) 2024-04-26T04:56:30.454 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.object_map_resize 2024-04-26T04:56:30.459 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.object_map_resize (5 ms) 2024-04-26T04:56:30.459 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.object_map_update 2024-04-26T04:56:30.466 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.object_map_update (7 ms) 2024-04-26T04:56:30.466 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2024-04-26T04:56:30.467 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.object_map_load_enoent (1 ms) 2024-04-26T04:56:30.467 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2024-04-26T04:56:30.472 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.object_map_snap_add (5 ms) 2024-04-26T04:56:30.472 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2024-04-26T04:56:30.479 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.object_map_snap_remove (7 ms) 2024-04-26T04:56:30.479 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.flags 2024-04-26T04:56:30.483 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.flags (5 ms) 2024-04-26T04:56:30.483 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.metadata 2024-04-26T04:56:30.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:30 smithi116 ceph-mon[29897]: osdmap e72: 8 total, 8 up, 8 in 2024-04-26T04:56:30.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:30 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/3774147553' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi012-77771-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:56:30.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:30 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/3774147553' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi012-77771-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-04-26T04:56:30.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:30 smithi116 ceph-mon[29897]: osdmap e73: 8 total, 8 up, 8 in 2024-04-26T04:56:30.505 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.metadata (21 ms) 2024-04-26T04:56:30.539 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.set_features 2024-04-26T04:56:30.540 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.set_features (7 ms) 2024-04-26T04:56:30.540 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.mirror 2024-04-26T04:56:30.548 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.mirror (36 ms) 2024-04-26T04:56:30.548 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.mirror_image 2024-04-26T04:56:30.565 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.mirror_image (17 ms) 2024-04-26T04:56:30.565 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.mirror_image_status 2024-04-26T04:56:31.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:31 smithi116 ceph-mon[29897]: pgmap v95: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:31.512 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:31 smithi012 ceph-mon[26617]: pgmap v95: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:31.512 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:31 smithi012 ceph-mon[37567]: pgmap v95: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:32.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:32 smithi116 ceph-mon[29897]: osdmap e74: 8 total, 8 up, 8 in 2024-04-26T04:56:32.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:32 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/4204201749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:56:32.512 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:32 smithi012 ceph-mon[26617]: osdmap e74: 8 total, 8 up, 8 in 2024-04-26T04:56:32.512 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:32 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/4204201749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:56:32.512 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:32 smithi012 ceph-mon[37567]: osdmap e74: 8 total, 8 up, 8 in 2024-04-26T04:56:32.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:32 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/4204201749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:56:32.830 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.mirror_image_status (2265 ms) 2024-04-26T04:56:32.831 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.mirror_image_map 2024-04-26T04:56:32.912 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.mirror_image_map (82 ms) 2024-04-26T04:56:32.912 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.mirror_instances 2024-04-26T04:56:32.920 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.mirror_instances (8 ms) 2024-04-26T04:56:32.920 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2024-04-26T04:56:32.932 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.mirror_snapshot (12 ms) 2024-04-26T04:56:32.932 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_dir_list 2024-04-26T04:56:32.934 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_dir_list (2 ms) 2024-04-26T04:56:32.934 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_dir_add 2024-04-26T04:56:32.936 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_dir_add (2 ms) 2024-04-26T04:56:32.936 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2024-04-26T04:56:32.939 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2024-04-26T04:56:32.939 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_dir_rename 2024-04-26T04:56:32.944 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_dir_rename (6 ms) 2024-04-26T04:56:32.945 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_dir_remove 2024-04-26T04:56:32.948 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_dir_remove (3 ms) 2024-04-26T04:56:32.948 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2024-04-26T04:56:32.951 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (3 ms) 2024-04-26T04:56:32.951 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_image_add 2024-04-26T04:56:32.953 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_image_add (2 ms) 2024-04-26T04:56:32.953 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_image_remove 2024-04-26T04:56:32.956 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_image_remove (3 ms) 2024-04-26T04:56:32.956 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_image_list 2024-04-26T04:56:32.959 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_image_list (3 ms) 2024-04-26T04:56:32.959 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_image_clean 2024-04-26T04:56:32.963 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_image_clean (5 ms) 2024-04-26T04:56:32.963 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.image_group_add 2024-04-26T04:56:32.965 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.image_group_add (2 ms) 2024-04-26T04:56:32.966 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.image_group_remove 2024-04-26T04:56:32.969 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.image_group_remove (3 ms) 2024-04-26T04:56:32.969 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.image_group_get 2024-04-26T04:56:32.971 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.image_group_get (2 ms) 2024-04-26T04:56:32.971 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2024-04-26T04:56:32.973 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2024-04-26T04:56:32.973 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2024-04-26T04:56:32.975 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2024-04-26T04:56:32.975 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2024-04-26T04:56:32.977 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (3 ms) 2024-04-26T04:56:32.977 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2024-04-26T04:56:32.980 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (2 ms) 2024-04-26T04:56:32.980 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_snap_set 2024-04-26T04:56:32.982 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_snap_set (2 ms) 2024-04-26T04:56:32.982 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_snap_list 2024-04-26T04:56:32.986 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_snap_list (4 ms) 2024-04-26T04:56:32.986 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2024-04-26T04:56:33.001 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (16 ms) 2024-04-26T04:56:33.002 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2024-04-26T04:56:33.026 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:56:32 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:56:32] "GET /metrics HTTP/1.1" 200 239100 "" "Prometheus/2.43.0" 2024-04-26T04:56:33.183 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (181 ms) 2024-04-26T04:56:33.183 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_snap_remove 2024-04-26T04:56:33.186 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_snap_remove (3 ms) 2024-04-26T04:56:33.186 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2024-04-26T04:56:33.188 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (3 ms) 2024-04-26T04:56:33.188 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.trash_methods 2024-04-26T04:56:33.196 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.trash_methods (7 ms) 2024-04-26T04:56:33.196 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.op_features 2024-04-26T04:56:33.202 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.op_features (6 ms) 2024-04-26T04:56:33.202 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.clone_parent 2024-04-26T04:56:33.222 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.clone_parent (20 ms) 2024-04-26T04:56:33.222 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2024-04-26T04:56:33.230 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.clone_parent_ns (8 ms) 2024-04-26T04:56:33.230 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.clone_child 2024-04-26T04:56:33.243 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.clone_child (13 ms) 2024-04-26T04:56:33.243 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.namespace_methods 2024-04-26T04:56:33.251 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.namespace_methods (8 ms) 2024-04-26T04:56:33.251 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.migration 2024-04-26T04:56:33.260 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.migration (9 ms) 2024-04-26T04:56:33.260 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.migration_v1 2024-04-26T04:56:33.273 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.migration_v1 (13 ms) 2024-04-26T04:56:33.273 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2024-04-26T04:56:33.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:33 smithi116 ceph-mon[29897]: pgmap v97: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:33.512 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:33 smithi012 ceph-mon[26617]: pgmap v97: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:33.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:33 smithi012 ceph-mon[37567]: pgmap v97: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:33.991 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:56:33 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:56:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:56:34.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:34 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/3947417307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:56:34.512 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:34 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/3947417307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:56:34.512 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:34 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/3947417307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:56:35.045 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1772 ms) 2024-04-26T04:56:35.046 INFO:tasks.workunit.client.0.smithi012.stdout:[ RUN ] TestClsRbd.sparsify 2024-04-26T04:56:35.078 INFO:tasks.workunit.client.0.smithi012.stdout:[ OK ] TestClsRbd.sparsify (33 ms) 2024-04-26T04:56:35.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:35 smithi116 ceph-mon[29897]: osdmap e75: 8 total, 8 up, 8 in 2024-04-26T04:56:35.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:35 smithi116 ceph-mon[29897]: pgmap v99: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.6 MiB/s wr, 2.70k op/s 2024-04-26T04:56:35.512 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:35 smithi012 ceph-mon[26617]: osdmap e75: 8 total, 8 up, 8 in 2024-04-26T04:56:35.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:35 smithi012 ceph-mon[26617]: pgmap v99: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.6 MiB/s wr, 2.70k op/s 2024-04-26T04:56:35.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:35 smithi012 ceph-mon[37567]: osdmap e75: 8 total, 8 up, 8 in 2024-04-26T04:56:35.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:35 smithi012 ceph-mon[37567]: pgmap v99: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.6 MiB/s wr, 2.70k op/s 2024-04-26T04:56:36.062 INFO:tasks.workunit.client.0.smithi012.stdout:[----------] 70 tests from TestClsRbd (5070 ms total) 2024-04-26T04:56:36.062 INFO:tasks.workunit.client.0.smithi012.stdout: 2024-04-26T04:56:36.062 INFO:tasks.workunit.client.0.smithi012.stdout:[----------] Global test environment tear-down 2024-04-26T04:56:36.062 INFO:tasks.workunit.client.0.smithi012.stdout:[==========] 70 tests from 1 test suite ran. (7634 ms total) 2024-04-26T04:56:36.063 INFO:tasks.workunit.client.0.smithi012.stdout:[ PASSED ] 70 tests. 2024-04-26T04:56:36.064 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-04-26T04:56:36.065 DEBUG:teuthology.orchestra.run.smithi012:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-04-26T04:56:36.133 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2024-04-26T04:56:36.133 DEBUG:teuthology.orchestra.run.smithi012:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-04-26T04:56:36.378 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:36 smithi012 ceph-mon[26617]: osdmap e76: 8 total, 8 up, 8 in 2024-04-26T04:56:36.378 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:36 smithi012 ceph-mon[37567]: osdmap e76: 8 total, 8 up, 8 in 2024-04-26T04:56:36.379 DEBUG:teuthology.parallel:result is None 2024-04-26T04:56:36.380 DEBUG:teuthology.orchestra.run.smithi012:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2024-04-26T04:56:36.404 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2024-04-26T04:56:36.404 DEBUG:teuthology.orchestra.run.smithi012:> rmdir -- /home/ubuntu/cephtest/mnt.0 2024-04-26T04:56:36.466 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2024-04-26T04:56:36.466 INFO:teuthology.task.sequential:In sequential, running task print... 2024-04-26T04:56:36.467 INFO:teuthology.task.print:**** done cls/test_cls_rbd.sh 5-workload 2024-04-26T04:56:36.467 DEBUG:teuthology.parallel:result is None 2024-04-26T04:56:36.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:36 smithi116 ceph-mon[29897]: osdmap e76: 8 total, 8 up, 8 in 2024-04-26T04:56:37.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:37 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/1336705527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:56:37.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:37 smithi116 ceph-mon[29897]: osdmap e77: 8 total, 8 up, 8 in 2024-04-26T04:56:37.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:37 smithi116 ceph-mon[29897]: pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:37.512 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:37 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1336705527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:56:37.512 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:37 smithi012 ceph-mon[26617]: osdmap e77: 8 total, 8 up, 8 in 2024-04-26T04:56:37.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:37 smithi012 ceph-mon[26617]: pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:37.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:37 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/1336705527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:56:37.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:37 smithi012 ceph-mon[37567]: osdmap e77: 8 total, 8 up, 8 in 2024-04-26T04:56:37.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:37 smithi012 ceph-mon[37567]: pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:38.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:38 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:38.512 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:38 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:38.512 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:38 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:39.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:39 smithi116 ceph-mon[29897]: pgmap v103: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:39.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:39 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/1972020397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:56:39.512 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:39 smithi012 ceph-mon[26617]: pgmap v103: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:39.512 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:39 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1972020397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:56:39.512 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:39 smithi012 ceph-mon[37567]: pgmap v103: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:39.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:39 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/1972020397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:56:41.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:41 smithi012 ceph-mon[26617]: pgmap v104: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:41.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:41 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/2531191496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:56:41.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:41 smithi012 ceph-mon[37567]: pgmap v104: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:41.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:41 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/2531191496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:56:41.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:41 smithi116 ceph-mon[29897]: pgmap v104: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:41.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:41 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/2531191496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:56:43.262 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:56:42 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:56:42] "GET /metrics HTTP/1.1" 200 239889 "" "Prometheus/2.43.0" 2024-04-26T04:56:43.561 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:43 smithi012 ceph-mon[26617]: pgmap v105: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:43.561 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:43 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/2352136306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:56:43.562 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:43 smithi012 ceph-mon[37567]: pgmap v105: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:43.562 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:43 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/2352136306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:56:43.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:43 smithi116 ceph-mon[29897]: pgmap v105: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:43.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:43 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/2352136306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:56:43.992 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:56:43 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:56:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:56:45.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:45 smithi012 ceph-mon[26617]: pgmap v106: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:45.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:45 smithi012 ceph-mon[37567]: pgmap v106: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:45.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:45 smithi116 ceph-mon[29897]: pgmap v106: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:46.765 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:46 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1844642458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:56:46.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:46 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/1844642458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:56:46.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:46 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/1844642458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:56:47.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:47 smithi012 ceph-mon[26617]: pgmap v107: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:47.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:47 smithi012 ceph-mon[37567]: pgmap v107: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:47.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:47 smithi116 ceph-mon[29897]: pgmap v107: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:48.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:48 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/2138439772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:56:49.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:48 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/2138439772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:56:49.012 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:48 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/2138439772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:56:49.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:49 smithi116 ceph-mon[29897]: pgmap v108: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:50.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:49 smithi012 ceph-mon[26617]: pgmap v108: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:50.012 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:49 smithi012 ceph-mon[37567]: pgmap v108: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:51.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:51 smithi012 ceph-mon[26617]: pgmap v109: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:51.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:51 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/541889106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:56:51.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:51 smithi012 ceph-mon[37567]: pgmap v109: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:51.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:51 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/541889106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:56:51.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:51 smithi116 ceph-mon[29897]: pgmap v109: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:51.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:51 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/541889106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:56:53.218 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:56:52 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:56:52] "GET /metrics HTTP/1.1" 200 239967 "" "Prometheus/2.43.0" 2024-04-26T04:56:53.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:53 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:56:53.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:53 smithi116 ceph-mon[29897]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (unknown) 2024-04-26T04:56:53.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:53 smithi116 ceph-mon[29897]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-26T04:56:53.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:53 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:56:53.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:53 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:56:53.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:53 smithi116 ceph-mon[29897]: Upgrade: Need to upgrade myself (mgr.y) 2024-04-26T04:56:53.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:53 smithi116 ceph-mon[29897]: pgmap v110: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:53.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:53 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:53.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:53 smithi116 ceph-mon[29897]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 on smithi116 2024-04-26T04:56:53.512 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:53 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:56:53.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:53 smithi012 ceph-mon[37567]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (unknown) 2024-04-26T04:56:53.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:53 smithi012 ceph-mon[37567]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-26T04:56:53.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:53 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:56:53.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:53 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:56:53.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:53 smithi012 ceph-mon[37567]: Upgrade: Need to upgrade myself (mgr.y) 2024-04-26T04:56:53.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:53 smithi012 ceph-mon[37567]: pgmap v110: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:53.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:53 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:53.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:53 smithi012 ceph-mon[37567]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 on smithi116 2024-04-26T04:56:53.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:53 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:56:53.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:53 smithi012 ceph-mon[26617]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (unknown) 2024-04-26T04:56:53.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:53 smithi012 ceph-mon[26617]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-26T04:56:53.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:53 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:56:53.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:53 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:56:53.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:53 smithi012 ceph-mon[26617]: Upgrade: Need to upgrade myself (mgr.y) 2024-04-26T04:56:53.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:53 smithi012 ceph-mon[26617]: pgmap v110: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:53.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:53 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:53.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:53 smithi012 ceph-mon[26617]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 on smithi116 2024-04-26T04:56:53.991 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:56:53 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:56:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:56:54.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:54 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/935536818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:56:54.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:54 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/935536818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:56:54.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:54 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/935536818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:56:55.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:55 smithi116 ceph-mon[29897]: pgmap v111: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:55.512 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:55 smithi012 ceph-mon[26617]: pgmap v111: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:55.512 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:55 smithi012 ceph-mon[37567]: pgmap v111: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:56.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:56 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/2706104333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:56:56.512 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:56 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/2706104333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:56:56.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:56 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/2706104333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:56:57.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:57 smithi116 ceph-mon[29897]: pgmap v112: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:57.512 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:57 smithi012 ceph-mon[37567]: pgmap v112: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:57.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:57 smithi012 ceph-mon[26617]: pgmap v112: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:58.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:58 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/137401855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:56:58.512 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:58 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/137401855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:56:58.512 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:58 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/137401855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:56:59.512 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:56:59 smithi012 ceph-mon[26617]: pgmap v113: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:59.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:56:59 smithi012 ceph-mon[37567]: pgmap v113: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:56:59.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:56:59 smithi116 ceph-mon[29897]: pgmap v113: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:00.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:00 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/1009165864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:00.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:00 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1009165864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:00.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:00 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/1009165864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:01.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:01 smithi012 ceph-mon[26617]: pgmap v114: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:01.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:01 smithi012 ceph-mon[37567]: pgmap v114: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:01.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:01 smithi116 ceph-mon[29897]: pgmap v114: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:03.262 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:02 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:57:02] "GET /metrics HTTP/1.1" 200 239967 "" "Prometheus/2.43.0" 2024-04-26T04:57:03.762 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:03 smithi012 ceph-mon[37567]: pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:03.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:03 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/421143069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:03.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:03 smithi012 ceph-mon[26617]: pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:03.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:03 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/421143069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:03.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:03 smithi116 ceph-mon[29897]: pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:03.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:03 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/421143069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:03.991 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:03 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:57:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:57:05.762 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:05 smithi012 ceph-mon[37567]: pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:05.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:05 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/2022238279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:05.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:05 smithi012 ceph-mon[26617]: pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:05.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:05 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/2022238279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:05.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:05 smithi116 ceph-mon[29897]: pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:05.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:05 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/2022238279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:07.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:07 smithi012 ceph-mon[26617]: pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:07.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:07 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:07.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:07 smithi012 ceph-mon[37567]: pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:07.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:07 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:07.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:07 smithi116 ceph-mon[29897]: pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:07.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:07 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:08.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:08 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/3292934151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:08.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:08 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/3292934151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:08.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:08 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/3292934151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:09.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:09 smithi012 ceph-mon[26617]: pgmap v118: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:09.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:09 smithi012 ceph-mon[37567]: pgmap v118: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:09.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:09 smithi116 ceph-mon[29897]: pgmap v118: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:10.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:10 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/2201295074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:11.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:10 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/2201295074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:11.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:10 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/2201295074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:11.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:11 smithi116 ceph-mon[29897]: pgmap v119: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:12.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:11 smithi012 ceph-mon[26617]: pgmap v119: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:12.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:11 smithi012 ceph-mon[37567]: pgmap v119: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:12.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:12 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/3532292089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:13.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:12 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/3532292089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:13.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:12 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/3532292089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:13.013 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:12 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:57:12] "GET /metrics HTTP/1.1" 200 239937 "" "Prometheus/2.43.0" 2024-04-26T04:57:13.914 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:13 smithi012 ceph-mon[26617]: pgmap v120: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:13.914 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:13 smithi012 ceph-mon[37567]: pgmap v120: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:13.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:13 smithi116 ceph-mon[29897]: pgmap v120: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:13.991 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:13 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:57:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:57:15.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:15 smithi116 ceph-mon[29897]: pgmap v121: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:15.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:15 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/2807131822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:16.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:15 smithi012 ceph-mon[26617]: pgmap v121: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:16.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:15 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/2807131822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:16.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:15 smithi012 ceph-mon[37567]: pgmap v121: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:16.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:15 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/2807131822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:17.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:17 smithi116 ceph-mon[29897]: pgmap v122: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:17.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:17 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/3740510942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:18.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:17 smithi012 ceph-mon[26617]: pgmap v122: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:18.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:17 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/3740510942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:18.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:17 smithi012 ceph-mon[37567]: pgmap v122: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:18.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:17 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/3740510942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:19.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:19 smithi116 ceph-mon[29897]: pgmap v123: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:20.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:19 smithi012 ceph-mon[26617]: pgmap v123: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:20.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:19 smithi012 ceph-mon[37567]: pgmap v123: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:20.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:20 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/1407571794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:21.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:20 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1407571794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:21.012 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:20 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/1407571794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:21.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:21 smithi116 ceph-mon[29897]: pgmap v124: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:22.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:21 smithi012 ceph-mon[26617]: pgmap v124: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:22.012 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:21 smithi012 ceph-mon[37567]: pgmap v124: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:22.807 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:22 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/2380479864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:22.807 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:22 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:57:22.807 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:22 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:57:22.807 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:22 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:57:22.807 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:22 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:57:22.807 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:22 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:57:22.808 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:22 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:23.013 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:22 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:57:22] "GET /metrics HTTP/1.1" 200 239916 "" "Prometheus/2.43.0" 2024-04-26T04:57:23.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:22 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/2380479864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:23.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:22 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:57:23.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:22 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:57:23.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:22 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:57:23.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:22 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:57:23.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:22 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:57:23.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:22 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:23.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:22 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/2380479864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:23.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:22 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:57:23.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:22 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:57:23.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:22 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:57:23.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:22 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:57:23.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:22 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:57:23.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:22 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:23.905 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:23 smithi116 ceph-mon[29897]: Upgrade: Updating mgr.x 2024-04-26T04:57:23.906 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:23 smithi116 ceph-mon[29897]: Deploying daemon mgr.x on smithi116 2024-04-26T04:57:23.906 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:23 smithi116 ceph-mon[29897]: pgmap v125: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:23.906 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:23 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[31810]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:57:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:57:24.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:23 smithi012 ceph-mon[26617]: Upgrade: Updating mgr.x 2024-04-26T04:57:24.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:23 smithi012 ceph-mon[26617]: Deploying daemon mgr.x on smithi116 2024-04-26T04:57:24.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:23 smithi012 ceph-mon[26617]: pgmap v125: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:24.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:23 smithi012 ceph-mon[37567]: Upgrade: Updating mgr.x 2024-04-26T04:57:24.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:23 smithi012 ceph-mon[37567]: Deploying daemon mgr.x on smithi116 2024-04-26T04:57:24.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:23 smithi012 ceph-mon[37567]: pgmap v125: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:24.241 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:24 smithi116 systemd[1]: Stopping Ceph mgr.x for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:57:24.693 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:24 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/3033306032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:24.693 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:24 smithi116 podman[62982]: 2024-04-26 04:57:24.414289257 +0000 UTC m=+0.139054900 container died dd1f1c263e39cebcaccfdd511a0ede6634c6eeda321b0af624487a37898404ac (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x, GIT_BRANCH=HEAD, RELEASE=quincy-c61472c, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-04-26T04:57:24.991 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:24 smithi116 podman[62982]: 2024-04-26 04:57:24.891966641 +0000 UTC m=+0.616732326 container cleanup dd1f1c263e39cebcaccfdd511a0ede6634c6eeda321b0af624487a37898404ac (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=quincy-c61472c, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:57:24.991 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:24 smithi116 bash[62982]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x 2024-04-26T04:57:25.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:24 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/3033306032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:25.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:24 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/3033306032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:25.491 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:25 smithi116 podman[62996]: 2024-04-26 04:57:25.177331175 +0000 UTC m=+0.767447490 container remove dd1f1c263e39cebcaccfdd511a0ede6634c6eeda321b0af624487a37898404ac (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, RELEASE=quincy-c61472c, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-04-26T04:57:25.491 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:25 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-26T04:57:25.491 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:25 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-04-26T04:57:25.491 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:25 smithi116 systemd[1]: Stopped Ceph mgr.x for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:57:25.492 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:25 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mgr.x.service: Consumed 17.195s CPU time. 2024-04-26T04:57:25.893 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:25 smithi116 ceph-mon[29897]: pgmap v126: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:25.893 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:25 smithi116 systemd[1]: Starting Ceph mgr.x for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:57:25.893 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:25 smithi116 podman[63122]: 2024-04-26 04:57:25.640446039 +0000 UTC m=+0.018038971 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:57:25.893 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:25 smithi116 podman[63122]: 2024-04-26 04:57:25.745208483 +0000 UTC m=+0.122801456 container create b8c790c0f9e49defff228c4aba835a009bf72383c141a7b8a4cec76c3dfd3d40 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True) 2024-04-26T04:57:26.012 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:25 smithi012 ceph-mon[37567]: pgmap v126: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:26.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:25 smithi012 ceph-mon[26617]: pgmap v126: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:26.197 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:26 smithi116 podman[63122]: 2024-04-26 04:57:26.039729939 +0000 UTC m=+0.417322863 container init b8c790c0f9e49defff228c4aba835a009bf72383c141a7b8a4cec76c3dfd3d40 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x, io.buildah.version=1.33.5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424) 2024-04-26T04:57:26.197 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:26 smithi116 podman[63122]: 2024-04-26 04:57:26.042822982 +0000 UTC m=+0.420415895 container start b8c790c0f9e49defff228c4aba835a009bf72383c141a7b8a4cec76c3dfd3d40 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-04-26T04:57:26.197 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:26 smithi116 ceph-mgr[63141]: -- 172.21.15.116:0/2126181841 <== mon.1 v2:172.21.15.116:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d9e67685a0 con 0x55d9e673c800 2024-04-26T04:57:26.197 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:26 smithi116 bash[63122]: b8c790c0f9e49defff228c4aba835a009bf72383c141a7b8a4cec76c3dfd3d40 2024-04-26T04:57:26.197 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:26 smithi116 systemd[1]: Started Ceph mgr.x for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:57:26.491 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:57:26.196+0000 7f8c6ae13200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:57:26.492 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:57:26.389+0000 7f8c6ae13200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:57:26.829 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:57:26.504+0000 7f8c6ae13200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:57:27.179 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:57:26.830+0000 7f8c6ae13200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:57:27.180 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:57:26.918+0000 7f8c6ae13200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:57:27.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:27 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:57:27.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:27 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:57:27.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:27 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:57:27.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:27 smithi116 ceph-mon[29897]: pgmap v127: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:27.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:27 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/59493411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:27.512 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:27 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:57:27.512 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:27 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:57:27.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:27 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:57:27.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:27 smithi012 ceph-mon[37567]: pgmap v127: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:27.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:27 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/59493411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:27.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:27 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:57:27.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:27 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:57:27.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:27 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:57:27.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:27 smithi012 ceph-mon[26617]: pgmap v127: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:27.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:27 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/59493411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:28.491 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:28 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:57:28.209+0000 7f8c6ae13200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:57:28.491 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:28 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:57:28.408+0000 7f8c6ae13200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:57:28.888 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:28 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:57:28.586+0000 7f8c6ae13200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:57:29.200 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:28 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:57:28.887+0000 7f8c6ae13200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:57:29.200 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:28 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:57:28.982+0000 7f8c6ae13200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:57:29.200 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:29 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:57:29.199+0000 7f8c6ae13200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:57:29.471 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:29 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:57:29.401+0000 7f8c6ae13200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:57:29.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:29 smithi116 ceph-mon[29897]: pgmap v128: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:29.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:29 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:57:29.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:29 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:57:29.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:29 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/2858302178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:29.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:29 smithi012 ceph-mon[26617]: pgmap v128: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:29.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:29 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:57:29.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:29 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:57:29.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:29 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/2858302178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:29.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:29 smithi012 ceph-mon[37567]: pgmap v128: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:29.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:29 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:57:29.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:29 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:57:29.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:29 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/2858302178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:30.491 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:30 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:57:30.055+0000 7f8c6ae13200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:57:30.741 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:30 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:57:30.490+0000 7f8c6ae13200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:57:30.742 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:30 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:57:30.597+0000 7f8c6ae13200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:57:31.241 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:30 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:57:30.812+0000 7f8c6ae13200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:57:31.673 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:31 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:57:31.369+0000 7f8c6ae13200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:57:31.673 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:31 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:57:31.470+0000 7f8c6ae13200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:57:31.673 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:31 smithi116 ceph-mon[29897]: pgmap v129: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:31.673 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:31 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:57:31.674 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:31 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:57:31.674 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:31 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:57:31.674 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:31 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:57:31.979 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:31 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:57:31.736+0000 7f8c6ae13200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:57:31.979 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:31 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:57:31.874+0000 7f8c6ae13200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:57:31.979 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:31 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:57:31.977+0000 7f8c6ae13200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:57:32.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:31 smithi012 ceph-mon[26617]: pgmap v129: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:32.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:31 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:57:32.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:31 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:57:32.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:31 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:57:32.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:31 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:57:32.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:31 smithi012 ceph-mon[37567]: pgmap v129: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:32.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:31 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:57:32.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:31 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:57:32.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:31 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:57:32.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:31 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:57:32.241 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:32 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:57:32.096+0000 7f8c6ae13200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:57:32.241 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:32 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:57:32.191+0000 7f8c6ae13200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:57:32.869 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:32 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/3065964802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:33.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:32 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/3065964802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:33.013 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:32 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:57:32] "GET /metrics HTTP/1.1" 200 239916 "" "Prometheus/2.43.0" 2024-04-26T04:57:33.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:32 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/3065964802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:33.205 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:33 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:57:33.031+0000 7f8c6ae13200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:57:33.491 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:33 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:57:33.257+0000 7f8c6ae13200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:57:33.491 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:33 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: [26/Apr/2024:04:57:33] ENGINE Bus STARTING 2024-04-26T04:57:33.492 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:33 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: CherryPy Checker: 2024-04-26T04:57:33.492 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:33 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: The Application mounted at '' has an empty config. 2024-04-26T04:57:33.492 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:33 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:57:33.492 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:33 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: [26/Apr/2024:04:57:33] ENGINE Serving on http://:::9283 2024-04-26T04:57:33.492 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:33 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: [26/Apr/2024:04:57:33] ENGINE Bus STARTED 2024-04-26T04:57:33.882 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:33 smithi116 ceph-mon[29897]: pgmap v130: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:33.882 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:33 smithi116 ceph-mon[29897]: Standby manager daemon x restarted 2024-04-26T04:57:33.882 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:33 smithi116 ceph-mon[29897]: Standby manager daemon x started 2024-04-26T04:57:33.882 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:33 smithi116 ceph-mon[29897]: from='mgr.? 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:57:33.882 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:33 smithi116 ceph-mon[29897]: from='mgr.? 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:57:33.882 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:33 smithi116 ceph-mon[29897]: from='mgr.? 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:57:33.882 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:33 smithi116 ceph-mon[29897]: from='mgr.? 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:57:33.882 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:33 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:57:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:57:34.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:33 smithi012 ceph-mon[26617]: pgmap v130: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:34.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:33 smithi012 ceph-mon[26617]: Standby manager daemon x restarted 2024-04-26T04:57:34.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:33 smithi012 ceph-mon[26617]: Standby manager daemon x started 2024-04-26T04:57:34.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:33 smithi012 ceph-mon[26617]: from='mgr.? 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:57:34.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:33 smithi012 ceph-mon[26617]: from='mgr.? 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:57:34.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:33 smithi012 ceph-mon[26617]: from='mgr.? 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:57:34.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:33 smithi012 ceph-mon[26617]: from='mgr.? 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:57:34.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:33 smithi012 ceph-mon[37567]: pgmap v130: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:34.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:33 smithi012 ceph-mon[37567]: Standby manager daemon x restarted 2024-04-26T04:57:34.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:33 smithi012 ceph-mon[37567]: Standby manager daemon x started 2024-04-26T04:57:34.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:33 smithi012 ceph-mon[37567]: from='mgr.? 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:57:34.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:33 smithi012 ceph-mon[37567]: from='mgr.? 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:57:34.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:33 smithi012 ceph-mon[37567]: from='mgr.? 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:57:34.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:33 smithi012 ceph-mon[37567]: from='mgr.? 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:57:34.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:34 smithi116 ceph-mon[29897]: mgrmap e21: y(active, since 3m), standbys: x 2024-04-26T04:57:34.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:34 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/2296699985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:35.012 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:34 smithi012 ceph-mon[37567]: mgrmap e21: y(active, since 3m), standbys: x 2024-04-26T04:57:35.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:34 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/2296699985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:35.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:34 smithi012 ceph-mon[26617]: mgrmap e21: y(active, since 3m), standbys: x 2024-04-26T04:57:35.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:34 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/2296699985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:35.373 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: [26/Apr/2024:04:57:35] ENGINE Bus STOPPING 2024-04-26T04:57:35.373 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: [26/Apr/2024:04:57:35] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:57:35.373 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: [26/Apr/2024:04:57:35] ENGINE Bus STOPPED 2024-04-26T04:57:35.621 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:57:35.294+0000 7fdba33dd700 -1 mgr handle_mgr_map I was active but no longer am 2024-04-26T04:57:35.621 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: ignoring --setuser ceph since I am not root 2024-04-26T04:57:35.622 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: ignoring --setgroup ceph since I am not root 2024-04-26T04:57:35.622 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mgr[26852]: -- 172.21.15.12:0/1280722052 <== mon.2 v2:172.21.15.12:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5556d3c789c0 con 0x5556d3c4e800 2024-04-26T04:57:35.622 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:57:35.467+0000 7fb19c06c1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:57:35.623 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: [26/Apr/2024:04:57:35] ENGINE Bus STARTING 2024-04-26T04:57:35.624 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:57:35.414+0000 7f8bcc205700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:57:35.624 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:57:35.414+0000 7f8bcc205700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:57:35.624 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:57:35.414+0000 7f8bcc205700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:57:35.624 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:57:35.414+0000 7f8bcc205700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:57:35.624 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:57:35.414+0000 7f8bcc205700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:57:35.624 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:57:35.414+0000 7f8bc89fe700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:57:35.624 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:57:35.414+0000 7f8bc89fe700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:57:35.624 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:57:35.414+0000 7f8bc89fe700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:57:35.624 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:57:35.414+0000 7f8bc89fe700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:57:35.624 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:57:35.414+0000 7f8bc89fe700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:57:35.624 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: [26/Apr/2024:04:57:35] ENGINE Serving on http://:::9283 2024-04-26T04:57:35.624 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: [26/Apr/2024:04:57:35] ENGINE Bus STARTED 2024-04-26T04:57:35.625 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: pgmap v131: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:35.625 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:57:35.625 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:57:35.625 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:57:35.625 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:57:35.625 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:57:35.625 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:57:35.625 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-26T04:57:35.625 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-26T04:57:35.625 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: Activating manager daemon x 2024-04-26T04:57:35.625 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: osdmap e78: 8 total, 8 up, 8 in 2024-04-26T04:57:35.625 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-04-26T04:57:35.625 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: mgrmap e22: x(active, starting, since 0.00672076s) 2024-04-26T04:57:35.625 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:57:35.626 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:57:35.626 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:57:35.626 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi012.fdqwmo"}]: dispatch 2024-04-26T04:57:35.626 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi116.kduyef"}]: dispatch 2024-04-26T04:57:35.626 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:57:35.626 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:57:35.626 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:57:35.626 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:57:35.626 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:57:35.626 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:57:35.626 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:57:35.626 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:57:35.626 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:57:35.626 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:57:35.627 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:57:35.627 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:57:35.627 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: Manager daemon x is now available 2024-04-26T04:57:35.627 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:57:35.627 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:35.627 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:57:35.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:57:35.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T04:57:35.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:35 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T04:57:36.012 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:57:35.636+0000 7fb19c06c1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:57:36.012 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:57:35.743+0000 7fb19c06c1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:57:36.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: pgmap v131: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:36.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:57:36.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:57:36.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:57:36.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:57:36.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:57:36.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:57:36.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-26T04:57:36.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-26T04:57:36.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: Activating manager daemon x 2024-04-26T04:57:36.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: osdmap e78: 8 total, 8 up, 8 in 2024-04-26T04:57:36.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-04-26T04:57:36.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: mgrmap e22: x(active, starting, since 0.00672076s) 2024-04-26T04:57:36.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:57:36.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:57:36.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:57:36.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi012.fdqwmo"}]: dispatch 2024-04-26T04:57:36.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi116.kduyef"}]: dispatch 2024-04-26T04:57:36.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:57:36.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:57:36.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:57:36.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:57:36.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:57:36.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:57:36.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:57:36.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:57:36.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:57:36.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:57:36.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:57:36.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:57:36.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: Manager daemon x is now available 2024-04-26T04:57:36.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:57:36.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:36.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:57:36.016 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:57:36.016 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T04:57:36.016 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T04:57:36.016 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: pgmap v131: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:36.016 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:57:36.016 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:57:36.016 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:57:36.016 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:57:36.016 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' 2024-04-26T04:57:36.016 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:57:36.016 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: from='mgr.14454 172.21.15.12:0/3840795289' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-26T04:57:36.017 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-26T04:57:36.017 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: Activating manager daemon x 2024-04-26T04:57:36.017 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: osdmap e78: 8 total, 8 up, 8 in 2024-04-26T04:57:36.017 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-04-26T04:57:36.017 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: mgrmap e22: x(active, starting, since 0.00672076s) 2024-04-26T04:57:36.017 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:57:36.017 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:57:36.017 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:57:36.017 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi012.fdqwmo"}]: dispatch 2024-04-26T04:57:36.017 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi116.kduyef"}]: dispatch 2024-04-26T04:57:36.017 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:57:36.017 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:57:36.017 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:57:36.017 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:57:36.017 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:57:36.018 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:57:36.018 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:57:36.018 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:57:36.018 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:57:36.018 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:57:36.018 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:57:36.018 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:57:36.018 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: Manager daemon x is now available 2024-04-26T04:57:36.018 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:57:36.018 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:36.018 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:57:36.018 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:57:36.018 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T04:57:36.018 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:35 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T04:57:36.512 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:36 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:57:36.068+0000 7fb19c06c1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:57:36.512 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:36 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:57:36.153+0000 7fb19c06c1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:57:37.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:36 smithi116 ceph-mon[29897]: Deploying cephadm binary to smithi116 2024-04-26T04:57:37.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:36 smithi116 ceph-mon[29897]: Deploying cephadm binary to smithi012 2024-04-26T04:57:37.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:36 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:37.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:36 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:37.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:36 smithi116 ceph-mon[29897]: mgrmap e23: x(active, since 1.00805s) 2024-04-26T04:57:37.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:36 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/2323892920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:37.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:36 smithi012 ceph-mon[37567]: Deploying cephadm binary to smithi116 2024-04-26T04:57:37.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:36 smithi012 ceph-mon[37567]: Deploying cephadm binary to smithi012 2024-04-26T04:57:37.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:36 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:37.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:36 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:37.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:36 smithi012 ceph-mon[37567]: mgrmap e23: x(active, since 1.00805s) 2024-04-26T04:57:37.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:36 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/2323892920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:37.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:36 smithi012 ceph-mon[26617]: Deploying cephadm binary to smithi116 2024-04-26T04:57:37.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:36 smithi012 ceph-mon[26617]: Deploying cephadm binary to smithi012 2024-04-26T04:57:37.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:36 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:37.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:36 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:37.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:36 smithi012 ceph-mon[26617]: mgrmap e23: x(active, since 1.00805s) 2024-04-26T04:57:37.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:36 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/2323892920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:37.752 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:37 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:57:37.404+0000 7fb19c06c1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:57:37.752 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:37 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:57:37.577+0000 7fb19c06c1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:57:38.012 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:37 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:57:37.751+0000 7fb19c06c1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:57:38.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:37 smithi012 ceph-mon[26617]: pgmap v3: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:38.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:37 smithi012 ceph-mon[26617]: [26/Apr/2024:04:57:36] ENGINE Bus STARTING 2024-04-26T04:57:38.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:37 smithi012 ceph-mon[26617]: [26/Apr/2024:04:57:36] ENGINE Serving on https://172.21.15.116:7150 2024-04-26T04:57:38.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:37 smithi012 ceph-mon[26617]: [26/Apr/2024:04:57:36] ENGINE Serving on http://172.21.15.116:8765 2024-04-26T04:57:38.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:37 smithi012 ceph-mon[26617]: [26/Apr/2024:04:57:36] ENGINE Bus STARTED 2024-04-26T04:57:38.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:37 smithi012 ceph-mon[37567]: pgmap v3: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:38.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:37 smithi012 ceph-mon[37567]: [26/Apr/2024:04:57:36] ENGINE Bus STARTING 2024-04-26T04:57:38.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:37 smithi012 ceph-mon[37567]: [26/Apr/2024:04:57:36] ENGINE Serving on https://172.21.15.116:7150 2024-04-26T04:57:38.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:37 smithi012 ceph-mon[37567]: [26/Apr/2024:04:57:36] ENGINE Serving on http://172.21.15.116:8765 2024-04-26T04:57:38.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:37 smithi012 ceph-mon[37567]: [26/Apr/2024:04:57:36] ENGINE Bus STARTED 2024-04-26T04:57:38.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:37 smithi116 ceph-mon[29897]: pgmap v3: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:38.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:37 smithi116 ceph-mon[29897]: [26/Apr/2024:04:57:36] ENGINE Bus STARTING 2024-04-26T04:57:38.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:37 smithi116 ceph-mon[29897]: [26/Apr/2024:04:57:36] ENGINE Serving on https://172.21.15.116:7150 2024-04-26T04:57:38.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:37 smithi116 ceph-mon[29897]: [26/Apr/2024:04:57:36] ENGINE Serving on http://172.21.15.116:8765 2024-04-26T04:57:38.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:37 smithi116 ceph-mon[29897]: [26/Apr/2024:04:57:36] ENGINE Bus STARTED 2024-04-26T04:57:38.508 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:38 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:57:38.026+0000 7fb19c06c1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:57:38.509 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:38 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:57:38.120+0000 7fb19c06c1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:57:38.509 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:38 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:57:38.218+0000 7fb19c06c1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:57:38.762 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:38 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:57:38.507+0000 7fb19c06c1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:57:39.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:38 smithi116 ceph-mon[29897]: pgmap v4: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:39.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:38 smithi116 ceph-mon[29897]: mgrmap e24: x(active, since 2s) 2024-04-26T04:57:39.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:38 smithi012 ceph-mon[26617]: pgmap v4: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:39.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:38 smithi012 ceph-mon[26617]: mgrmap e24: x(active, since 2s) 2024-04-26T04:57:39.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:38 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:57:38.993+0000 7fb19c06c1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:57:39.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:38 smithi012 ceph-mon[37567]: pgmap v4: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:39.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:38 smithi012 ceph-mon[37567]: mgrmap e24: x(active, since 2s) 2024-04-26T04:57:39.818 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:39 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:57:39.548+0000 7fb19c06c1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:57:39.818 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:39 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:57:39.647+0000 7fb19c06c1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:57:40.146 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:39 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1651527395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:40.147 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:39 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/1651527395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:40.147 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:39 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:57:39.837+0000 7fb19c06c1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:57:40.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:39 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/1651527395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:40.675 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:40 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:57:40.145+0000 7fb19c06c1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:57:40.675 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:40 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:57:40.413+0000 7fb19c06c1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:57:41.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:40 smithi012 ceph-mon[26617]: pgmap v5: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:41.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:40 smithi012 ceph-mon[26617]: mgrmap e25: x(active, since 4s) 2024-04-26T04:57:41.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:40 smithi012 ceph-mon[37567]: pgmap v5: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:41.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:40 smithi012 ceph-mon[37567]: mgrmap e25: x(active, since 4s) 2024-04-26T04:57:41.013 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:40 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:57:40.674+0000 7fb19c06c1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:57:41.013 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:40 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:57:40.817+0000 7fb19c06c1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:57:41.013 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:40 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:57:40.923+0000 7fb19c06c1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:57:41.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:40 smithi116 ceph-mon[29897]: pgmap v5: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:41.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:40 smithi116 ceph-mon[29897]: mgrmap e25: x(active, since 4s) 2024-04-26T04:57:41.439 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:41 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:57:41.059+0000 7fb19c06c1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:57:41.439 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:41 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:57:41.161+0000 7fb19c06c1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:57:42.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:41 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/3748547415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:42.249 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:41 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/3748547415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:42.249 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:41 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/3748547415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:42.250 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:42 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:57:42.036+0000 7fb19c06c1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:57:42.250 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:42 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:57:42.248+0000 7fb19c06c1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:57:42.513 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:42 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: [26/Apr/2024:04:57:42] ENGINE Bus STARTING 2024-04-26T04:57:42.513 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:42 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: CherryPy Checker: 2024-04-26T04:57:42.513 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:42 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: The Application mounted at '' has an empty config. 2024-04-26T04:57:42.513 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:42 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: 2024-04-26T04:57:42.513 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:42 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: [26/Apr/2024:04:57:42] ENGINE Serving on http://:::9283 2024-04-26T04:57:42.513 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:42 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: [26/Apr/2024:04:57:42] ENGINE Bus STARTED 2024-04-26T04:57:43.073 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:42 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:57:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:57:43.073 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:42 smithi012 ceph-mon[26617]: pgmap v6: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:43.073 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:42 smithi012 ceph-mon[26617]: Standby manager daemon y started 2024-04-26T04:57:43.074 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:42 smithi012 ceph-mon[26617]: from='mgr.? 172.21.15.12:0/8841837' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T04:57:43.074 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:42 smithi012 ceph-mon[26617]: from='mgr.? 172.21.15.12:0/8841837' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:57:43.074 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:42 smithi012 ceph-mon[26617]: from='mgr.? 172.21.15.12:0/8841837' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T04:57:43.074 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:42 smithi012 ceph-mon[26617]: from='mgr.? 172.21.15.12:0/8841837' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:57:43.074 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:42 smithi012 ceph-mon[37567]: pgmap v6: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:43.074 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:42 smithi012 ceph-mon[37567]: Standby manager daemon y started 2024-04-26T04:57:43.074 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:42 smithi012 ceph-mon[37567]: from='mgr.? 172.21.15.12:0/8841837' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T04:57:43.074 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:42 smithi012 ceph-mon[37567]: from='mgr.? 172.21.15.12:0/8841837' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:57:43.074 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:42 smithi012 ceph-mon[37567]: from='mgr.? 172.21.15.12:0/8841837' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T04:57:43.074 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:42 smithi012 ceph-mon[37567]: from='mgr.? 172.21.15.12:0/8841837' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:57:43.232 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:42 smithi116 ceph-mon[29897]: pgmap v6: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:43.232 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:42 smithi116 ceph-mon[29897]: Standby manager daemon y started 2024-04-26T04:57:43.232 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:42 smithi116 ceph-mon[29897]: from='mgr.? 172.21.15.12:0/8841837' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T04:57:43.232 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:42 smithi116 ceph-mon[29897]: from='mgr.? 172.21.15.12:0/8841837' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:57:43.232 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:42 smithi116 ceph-mon[29897]: from='mgr.? 172.21.15.12:0/8841837' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T04:57:43.232 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:42 smithi116 ceph-mon[29897]: from='mgr.? 172.21.15.12:0/8841837' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:57:43.840 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:43 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:57:43] "GET /metrics HTTP/1.1" 200 30859 "" "Prometheus/2.43.0" 2024-04-26T04:57:44.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:43 smithi012 ceph-mon[26617]: mgrmap e26: x(active, since 7s), standbys: y 2024-04-26T04:57:44.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:43 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:57:44.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:43 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:44.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:43 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:44.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:43 smithi012 ceph-mon[37567]: mgrmap e26: x(active, since 7s), standbys: y 2024-04-26T04:57:44.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:43 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:57:44.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:43 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:44.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:43 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:44.213 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:43 smithi116 ceph-mon[29897]: mgrmap e26: x(active, since 7s), standbys: y 2024-04-26T04:57:44.213 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:43 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:57:44.213 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:43 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:44.213 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:43 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:45.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:44 smithi012 ceph-mon[37567]: pgmap v7: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T04:57:45.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:44 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/1185507779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:45.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:44 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:45.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:44 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:45.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:44 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:45.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:44 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:45.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:44 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:45.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:44 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:45.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:44 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:45.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:44 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:45.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:44 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:45.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:44 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:45.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:44 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:45.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:44 smithi012 ceph-mon[26617]: pgmap v7: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T04:57:45.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:44 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1185507779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:45.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:44 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:45.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:44 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:45.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:44 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:45.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:44 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:45.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:44 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:45.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:44 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:45.015 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:44 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:45.015 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:44 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:45.015 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:44 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:45.015 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:44 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:45.015 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:44 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:45.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:44 smithi116 ceph-mon[29897]: pgmap v7: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T04:57:45.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:44 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/1185507779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:45.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:44 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:45.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:44 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:45.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:44 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:45.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:44 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:45.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:44 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:45.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:44 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:45.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:44 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:45.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:44 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:45.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:44 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:45.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:44 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:45.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:44 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:46.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:45 smithi116 ceph-mon[29897]: Adjusting osd_memory_target on smithi012 to 2481M 2024-04-26T04:57:46.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:45 smithi012 ceph-mon[26617]: Adjusting osd_memory_target on smithi012 to 2481M 2024-04-26T04:57:46.262 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:45 smithi012 ceph-mon[37567]: Adjusting osd_memory_target on smithi012 to 2481M 2024-04-26T04:57:47.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:46 smithi116 ceph-mon[29897]: pgmap v8: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:57:47.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:46 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/1591636743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:47.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:46 smithi012 ceph-mon[26617]: pgmap v8: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:57:47.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:46 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1591636743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:47.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:46 smithi012 ceph-mon[37567]: pgmap v8: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:57:47.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:46 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/1591636743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:49.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:48 smithi116 ceph-mon[29897]: pgmap v9: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-26T04:57:49.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:48 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/1460109344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:49.262 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:48 smithi012 ceph-mon[37567]: pgmap v9: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-26T04:57:49.262 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:48 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/1460109344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:49.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:48 smithi012 ceph-mon[26617]: pgmap v9: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-26T04:57:49.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:48 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1460109344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:51.130 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:50 smithi116 ceph-mon[29897]: pgmap v10: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:57:51.130 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:50 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:51.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:50 smithi012 ceph-mon[26617]: pgmap v10: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:57:51.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:50 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:51.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:50 smithi012 ceph-mon[37567]: pgmap v10: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:57:51.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:50 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:52.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:51 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/1209913202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:52.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:51 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1209913202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:52.262 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:51 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/1209913202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:53.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:52 smithi116 ceph-mon[29897]: pgmap v11: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:57:53.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:52 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:53.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:52 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:53.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:52 smithi012 ceph-mon[26617]: pgmap v11: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:57:53.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:52 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:53.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:52 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:53.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:52 smithi012 ceph-mon[37567]: pgmap v11: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:57:53.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:52 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:53.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:52 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:53.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:57:52 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:57:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:57:53.931 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:57:53 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:57:53] "GET /metrics HTTP/1.1" 200 33708 "" "Prometheus/2.43.0" 2024-04-26T04:57:54.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:54 smithi012 ceph-mon[26617]: pgmap v12: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:57:54.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:54 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:54.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:54 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:54.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:54 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/3251138354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:54.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:54 smithi012 ceph-mon[37567]: pgmap v12: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:57:54.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:54 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:54.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:54 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:54.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:54 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/3251138354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:54.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:54 smithi116 ceph-mon[29897]: pgmap v12: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:57:54.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:54 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:54.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:54 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:54.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:54 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/3251138354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:56.369 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:56 smithi116 ceph-mon[29897]: pgmap v13: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:56.370 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:56 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/3745958729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:56.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:56 smithi012 ceph-mon[26617]: pgmap v13: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:56.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:56 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/3745958729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:56.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:56 smithi012 ceph-mon[37567]: pgmap v13: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:56.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:56 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/3745958729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:58.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:57 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:58.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:57 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:58.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:57 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:58.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:57 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:58.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:57 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:58.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:57 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:58.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:57 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:58.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:57 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:58.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:57 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:58.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:57 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:58.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:57 smithi116 ceph-mon[29897]: Adjusting osd_memory_target on smithi116 to 2480M 2024-04-26T04:57:58.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:57 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:58.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:57 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:57:58.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:57 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:57:58.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:57 smithi116 ceph-mon[29897]: Updating smithi012:/etc/ceph/ceph.conf 2024-04-26T04:57:58.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:57 smithi116 ceph-mon[29897]: Updating smithi116:/etc/ceph/ceph.conf 2024-04-26T04:57:58.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:57 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:58.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:57 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:58.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:57 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:58.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:57 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:58.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:57 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:58.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:57 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:58.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:57 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:58.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:57 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:58.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:57 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:58.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:57 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:58.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:57 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:58.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:57 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:58.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:57 smithi012 ceph-mon[37567]: Adjusting osd_memory_target on smithi116 to 2480M 2024-04-26T04:57:58.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:57 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:58.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:57 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:57:58.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:57 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:57:58.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:57 smithi012 ceph-mon[37567]: Updating smithi012:/etc/ceph/ceph.conf 2024-04-26T04:57:58.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:57 smithi012 ceph-mon[37567]: Updating smithi116:/etc/ceph/ceph.conf 2024-04-26T04:57:58.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:57 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:58.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:57 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:58.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:57 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:58.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:57 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:58.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:57 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:58.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:57 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:58.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:57 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:58.265 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:57 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:58.265 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:57 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:58.265 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:57 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:58.265 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:57 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:58.265 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:57 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:57:58.265 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:57 smithi012 ceph-mon[26617]: Adjusting osd_memory_target on smithi116 to 2480M 2024-04-26T04:57:58.265 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:57 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:58.265 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:57 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:57:58.265 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:57 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:57:58.265 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:57 smithi012 ceph-mon[26617]: Updating smithi012:/etc/ceph/ceph.conf 2024-04-26T04:57:58.265 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:57 smithi012 ceph-mon[26617]: Updating smithi116:/etc/ceph/ceph.conf 2024-04-26T04:57:58.265 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:57 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:58.265 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:57 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:59.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:58 smithi116 ceph-mon[29897]: Updating smithi116:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:57:59.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:58 smithi116 ceph-mon[29897]: Updating smithi012:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:57:59.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:58 smithi116 ceph-mon[29897]: pgmap v14: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:59.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:58 smithi116 ceph-mon[29897]: Updating smithi116:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:57:59.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:58 smithi116 ceph-mon[29897]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:57:59.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:58 smithi116 ceph-mon[29897]: Updating smithi116:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:57:59.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:58 smithi116 ceph-mon[29897]: Updating smithi012:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:57:59.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:58 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:59.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:58 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:59.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:58 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:59.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:58 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:59.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:58 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:59.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:58 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:59.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:58 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:59.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:58 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/641053869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:59.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:58 smithi012 ceph-mon[26617]: Updating smithi116:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:57:59.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:58 smithi012 ceph-mon[26617]: Updating smithi012:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:57:59.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:58 smithi012 ceph-mon[26617]: pgmap v14: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:59.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:58 smithi012 ceph-mon[26617]: Updating smithi116:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:57:59.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:58 smithi012 ceph-mon[26617]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:57:59.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:58 smithi012 ceph-mon[26617]: Updating smithi116:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:57:59.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:58 smithi012 ceph-mon[26617]: Updating smithi012:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:57:59.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:58 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:59.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:58 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:59.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:58 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:59.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:58 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:59.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:58 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:59.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:58 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:59.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:58 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:59.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:58 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/641053869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:57:59.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:58 smithi012 ceph-mon[37567]: Updating smithi116:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:57:59.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:58 smithi012 ceph-mon[37567]: Updating smithi012:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:57:59.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:58 smithi012 ceph-mon[37567]: pgmap v14: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:57:59.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:58 smithi012 ceph-mon[37567]: Updating smithi116:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:57:59.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:58 smithi012 ceph-mon[37567]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:57:59.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:58 smithi012 ceph-mon[37567]: Updating smithi116:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:57:59.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:58 smithi012 ceph-mon[37567]: Updating smithi012:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:57:59.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:58 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:59.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:58 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:59.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:58 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:59.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:58 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:59.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:58 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:59.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:58 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:59.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:58 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:57:59.265 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:58 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/641053869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:00.077 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:59 smithi012 ceph-mon[26617]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-26T04:58:00.077 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:57:59 smithi012 ceph-mon[26617]: Reconfiguring daemon alertmanager.a on smithi012 2024-04-26T04:58:00.077 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:59 smithi012 ceph-mon[37567]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-26T04:58:00.077 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:57:59 smithi012 ceph-mon[37567]: Reconfiguring daemon alertmanager.a on smithi012 2024-04-26T04:58:00.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:59 smithi116 ceph-mon[29897]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-26T04:58:00.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:57:59 smithi116 ceph-mon[29897]: Reconfiguring daemon alertmanager.a on smithi012 2024-04-26T04:58:01.177 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:00 smithi012 ceph-mon[26617]: pgmap v15: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:01.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:00 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1355125933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:01.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:00 smithi012 ceph-mon[37567]: pgmap v15: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:01.178 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:00 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/1355125933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:01.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:00 smithi116 ceph-mon[29897]: pgmap v15: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:01.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:00 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/1355125933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:01.513 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:58:01 smithi012 systemd[1]: Stopping Ceph alertmanager.a for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:58:01.857 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:58:01 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[70793]: ts=2024-04-26T04:58:01.556Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-26T04:58:01.857 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:58:01 smithi012 podman[81567]: 2024-04-26 04:58:01.596645847 +0000 UTC m=+0.146933927 container died 10573377212bb2428986e5bcef8b447043432313ce97bc6890275f3975b01141 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:58:02.233 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:58:01 smithi012 podman[81567]: 2024-04-26 04:58:01.947897125 +0000 UTC m=+0.498185211 container cleanup 10573377212bb2428986e5bcef8b447043432313ce97bc6890275f3975b01141 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:58:02.233 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:58:01 smithi012 bash[81567]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a 2024-04-26T04:58:02.512 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:58:02 smithi012 podman[81581]: 2024-04-26 04:58:02.232560423 +0000 UTC m=+0.630791960 container remove 10573377212bb2428986e5bcef8b447043432313ce97bc6890275f3975b01141 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:58:02.513 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:58:02 smithi012 podman[81581]: 2024-04-26 04:58:02.32504464 +0000 UTC m=+0.723276180 volume remove e0544763e9393ad9e11fd75b51101139cda9af1f0c1b5dd32e248495fb7c1bad 2024-04-26T04:58:02.852 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:02 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[26848]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:58:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:58:02.852 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:58:02 smithi012 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-26T04:58:02.852 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:58:02 smithi012 systemd[1]: Stopped Ceph alertmanager.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:58:02.852 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:58:02 smithi012 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@alertmanager.a.service: Consumed 1.061s CPU time. 2024-04-26T04:58:02.853 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:58:02 smithi012 systemd[1]: Starting Ceph alertmanager.a for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:58:02.853 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:58:02 smithi012 podman[81679]: 2024-04-26 04:58:02.75117008 +0000 UTC m=+0.115150624 volume create a341212677031dae9e6b64a6b2a13d60cf3e420e820a4ffead8a01fcd4fc8daf 2024-04-26T04:58:02.853 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:58:02 smithi012 podman[81679]: 2024-04-26 04:58:02.654453031 +0000 UTC m=+0.018433592 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-04-26T04:58:02.853 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:58:02 smithi012 podman[81679]: 2024-04-26 04:58:02.85195864 +0000 UTC m=+0.215939230 container create 217bb19e153b57d1d56bfdea3f7251d164d284f57bca979ba20ae14265006d8a (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:58:03.138 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:02 smithi012 ceph-mon[26617]: pgmap v16: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:03.138 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:02 smithi012 ceph-mon[37567]: pgmap v16: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:03.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:02 smithi116 ceph-mon[29897]: pgmap v16: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:03.513 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:58:03 smithi012 podman[81679]: 2024-04-26 04:58:03.13809475 +0000 UTC m=+0.502075295 container init 217bb19e153b57d1d56bfdea3f7251d164d284f57bca979ba20ae14265006d8a (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:58:03.513 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:58:03 smithi012 podman[81679]: 2024-04-26 04:58:03.142017018 +0000 UTC m=+0.505997570 container start 217bb19e153b57d1d56bfdea3f7251d164d284f57bca979ba20ae14265006d8a (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:58:03.513 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:58:03 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[81716]: ts=2024-04-26T04:58:03.153Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-04-26T04:58:03.513 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:58:03 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[81716]: ts=2024-04-26T04:58:03.154Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-04-26T04:58:03.513 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:58:03 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[81716]: ts=2024-04-26T04:58:03.155Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.12 port=9094 2024-04-26T04:58:03.513 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:58:03 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[81716]: ts=2024-04-26T04:58:03.156Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-26T04:58:03.513 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:58:03 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[81716]: ts=2024-04-26T04:58:03.188Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-26T04:58:03.513 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:58:03 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[81716]: ts=2024-04-26T04:58:03.189Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-26T04:58:03.513 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:58:03 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[81716]: ts=2024-04-26T04:58:03.191Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-04-26T04:58:03.513 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:58:03 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[81716]: ts=2024-04-26T04:58:03.191Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-04-26T04:58:03.513 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:58:03 smithi012 bash[81679]: 217bb19e153b57d1d56bfdea3f7251d164d284f57bca979ba20ae14265006d8a 2024-04-26T04:58:03.513 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:58:03 smithi012 systemd[1]: Started Ceph alertmanager.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:58:03.991 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:03 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:58:03] "GET /metrics HTTP/1.1" 200 33708 "" "Prometheus/2.43.0" 2024-04-26T04:58:04.102 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:03 smithi012 systemd[1]: Stopping Ceph node-exporter.a for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:58:04.399 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:04 smithi012 podman[81862]: 2024-04-26 04:58:04.102561164 +0000 UTC m=+0.111785601 container died 991e63fb00765dd78747c8f1469e77ca5abdfa2cacc772d7198d3f015f54e714 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:58:04.400 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:04 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:04.400 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:04 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:04.400 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:04 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/689644043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:04.400 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:04 smithi012 ceph-mon[26617]: pgmap v17: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:04.400 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:04 smithi012 ceph-mon[26617]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-26T04:58:04.400 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:04 smithi012 ceph-mon[26617]: Reconfiguring daemon node-exporter.a on smithi012 2024-04-26T04:58:04.400 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:04 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:04.401 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:04 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:04.401 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:04 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/689644043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:04.401 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:04 smithi012 ceph-mon[37567]: pgmap v17: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:04.401 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:04 smithi012 ceph-mon[37567]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-26T04:58:04.401 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:04 smithi012 ceph-mon[37567]: Reconfiguring daemon node-exporter.a on smithi012 2024-04-26T04:58:04.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:04 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:04.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:04 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:04.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:04 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/689644043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:04.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:04 smithi116 ceph-mon[29897]: pgmap v17: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:04.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:04 smithi116 ceph-mon[29897]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-26T04:58:04.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:04 smithi116 ceph-mon[29897]: Reconfiguring daemon node-exporter.a on smithi012 2024-04-26T04:58:04.762 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:04 smithi012 podman[81862]: 2024-04-26 04:58:04.399817489 +0000 UTC m=+0.409041921 container cleanup 991e63fb00765dd78747c8f1469e77ca5abdfa2cacc772d7198d3f015f54e714 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:58:04.762 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:04 smithi012 bash[81862]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a 2024-04-26T04:58:05.157 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:04 smithi012 podman[81875]: 2024-04-26 04:58:04.768603614 +0000 UTC m=+0.670025461 container remove 991e63fb00765dd78747c8f1469e77ca5abdfa2cacc772d7198d3f015f54e714 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:58:05.157 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:04 smithi012 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-26T04:58:05.157 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:04 smithi012 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-26T04:58:05.157 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:04 smithi012 systemd[1]: Stopped Ceph node-exporter.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:58:05.157 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:04 smithi012 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@node-exporter.a.service: Consumed 2.142s CPU time. 2024-04-26T04:58:05.157 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:04 smithi012 systemd[1]: Starting Ceph node-exporter.a for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:58:05.414 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[81716]: ts=2024-04-26T04:58:05.156Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000689951s 2024-04-26T04:58:05.414 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 podman[81969]: 2024-04-26 04:58:05.203078567 +0000 UTC m=+0.113683434 container create 6390338999c1df79904119d62135930f153726a0f654122e1655880d620a1621 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:58:05.414 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 podman[81969]: 2024-04-26 04:58:05.107884271 +0000 UTC m=+0.018489150 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-26T04:58:05.414 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 podman[81969]: 2024-04-26 04:58:05.413733221 +0000 UTC m=+0.324338089 container init 6390338999c1df79904119d62135930f153726a0f654122e1655880d620a1621 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:58:05.763 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 podman[81969]: 2024-04-26 04:58:05.416592629 +0000 UTC m=+0.327197500 container start 6390338999c1df79904119d62135930f153726a0f654122e1655880d620a1621 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:58:05.763 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.419Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-26T04:58:05.763 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.419Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-26T04:58:05.763 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.420Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-26T04:58:05.763 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.420Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-26T04:58:05.763 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.420Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-26T04:58:05.764 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.420Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-26T04:58:05.764 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-26T04:58:05.764 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=arp 2024-04-26T04:58:05.764 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-26T04:58:05.764 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-26T04:58:05.764 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-26T04:58:05.764 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-26T04:58:05.764 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-26T04:58:05.764 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-26T04:58:05.764 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-26T04:58:05.764 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-26T04:58:05.765 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=edac 2024-04-26T04:58:05.765 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-26T04:58:05.765 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-26T04:58:05.765 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-26T04:58:05.765 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-26T04:58:05.765 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-26T04:58:05.765 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-26T04:58:05.765 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-26T04:58:05.765 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-26T04:58:05.765 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-26T04:58:05.765 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-26T04:58:05.766 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-26T04:58:05.766 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-26T04:58:05.766 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-26T04:58:05.766 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-26T04:58:05.766 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-26T04:58:05.766 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-26T04:58:05.766 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=os 2024-04-26T04:58:05.766 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-26T04:58:05.766 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-26T04:58:05.766 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-26T04:58:05.766 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-26T04:58:05.766 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-26T04:58:05.766 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-26T04:58:05.767 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-26T04:58:05.767 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=stat 2024-04-26T04:58:05.767 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-26T04:58:05.767 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-26T04:58:05.767 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-26T04:58:05.767 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=time 2024-04-26T04:58:05.767 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-26T04:58:05.767 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=uname 2024-04-26T04:58:05.767 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-26T04:58:05.767 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-26T04:58:05.767 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.421Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-26T04:58:05.767 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.422Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-26T04:58:05.767 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a[81983]: ts=2024-04-26T04:58:05.422Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-26T04:58:05.767 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 bash[81969]: 6390338999c1df79904119d62135930f153726a0f654122e1655880d620a1621 2024-04-26T04:58:05.767 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 04:58:05 smithi012 systemd[1]: Started Ceph node-exporter.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:58:06.609 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:06 smithi116 ceph-mon[29897]: pgmap v18: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:06.609 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:06 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:06.610 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:06 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:06.610 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:06 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:06.610 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:06 smithi116 ceph-mon[29897]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:58:06.610 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:06 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/3611349802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:06.610 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:06 smithi116 ceph-mon[29897]: Reconfiguring daemon prometheus.a on smithi116 2024-04-26T04:58:06.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:06 smithi012 ceph-mon[37567]: pgmap v18: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:06.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:06 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:06.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:06 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:06.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:06 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:06.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:06 smithi012 ceph-mon[37567]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:58:06.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:06 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/3611349802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:06.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:06 smithi012 ceph-mon[37567]: Reconfiguring daemon prometheus.a on smithi116 2024-04-26T04:58:06.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:06 smithi012 ceph-mon[26617]: pgmap v18: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:06.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:06 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:06.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:06 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:06.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:06 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:06.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:06 smithi012 ceph-mon[26617]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:58:06.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:06 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/3611349802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:06.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:06 smithi012 ceph-mon[26617]: Reconfiguring daemon prometheus.a on smithi116 2024-04-26T04:58:08.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:08 smithi116 ceph-mon[29897]: pgmap v19: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:08.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:08 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/2146720789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:08.762 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:08 smithi012 ceph-mon[37567]: pgmap v19: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:08.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:08 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/2146720789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:08.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:08 smithi012 ceph-mon[26617]: pgmap v19: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:08.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:08 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/2146720789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:09.348 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:09 smithi116 systemd[1]: Stopping Ceph prometheus.a for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:58:09.649 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:09 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[60337]: ts=2024-04-26T04:58:09.348Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-26T04:58:09.650 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:09 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[60337]: ts=2024-04-26T04:58:09.348Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-26T04:58:09.650 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:09 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[60337]: ts=2024-04-26T04:58:09.348Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-26T04:58:09.650 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:09 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[60337]: ts=2024-04-26T04:58:09.348Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-26T04:58:09.650 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:09 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[60337]: ts=2024-04-26T04:58:09.348Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-26T04:58:09.650 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:09 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[60337]: ts=2024-04-26T04:58:09.348Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-26T04:58:09.650 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:09 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[60337]: ts=2024-04-26T04:58:09.348Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-26T04:58:09.650 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:09 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[60337]: ts=2024-04-26T04:58:09.348Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-26T04:58:09.650 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:09 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[60337]: ts=2024-04-26T04:58:09.349Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-26T04:58:09.650 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:09 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[60337]: ts=2024-04-26T04:58:09.349Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-26T04:58:09.650 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:09 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[60337]: ts=2024-04-26T04:58:09.349Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-26T04:58:09.650 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:09 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[60337]: ts=2024-04-26T04:58:09.349Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-26T04:58:09.650 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:09 smithi116 podman[71562]: 2024-04-26 04:58:09.358844267 +0000 UTC m=+0.104221368 container died db82def51910508c2620309ad320fa7f1b770d2161d6ec43d334d326084f0ded (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:58:09.957 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:09 smithi116 podman[71562]: 2024-04-26 04:58:09.756588776 +0000 UTC m=+0.501965878 container cleanup db82def51910508c2620309ad320fa7f1b770d2161d6ec43d334d326084f0ded (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:58:09.957 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:09 smithi116 bash[71562]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a 2024-04-26T04:58:10.241 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:09 smithi116 podman[71577]: 2024-04-26 04:58:09.957433643 +0000 UTC m=+0.601313501 container remove db82def51910508c2620309ad320fa7f1b770d2161d6ec43d334d326084f0ded (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:58:10.241 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:10 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-26T04:58:10.241 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:10 smithi116 systemd[1]: Stopped Ceph prometheus.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:58:10.241 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:10 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 1.558s CPU time. 2024-04-26T04:58:10.242 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:10 smithi116 systemd[1]: Starting Ceph prometheus.a for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:58:10.660 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:10 smithi116 ceph-mon[29897]: pgmap v20: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:10.661 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:10 smithi116 podman[71673]: 2024-04-26 04:58:10.308663772 +0000 UTC m=+0.019268985 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-26T04:58:10.661 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:10 smithi116 podman[71673]: 2024-04-26 04:58:10.408287112 +0000 UTC m=+0.118892352 container create 8dbc41610354acb93cffbfb057018bfdf217bf2e63298919209d97511c696720 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:58:10.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:10 smithi012 ceph-mon[26617]: pgmap v20: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:10.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:10 smithi012 ceph-mon[37567]: pgmap v20: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:10.992 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:10 smithi116 podman[71673]: 2024-04-26 04:58:10.660652488 +0000 UTC m=+0.371257685 container init 8dbc41610354acb93cffbfb057018bfdf217bf2e63298919209d97511c696720 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:58:10.992 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:10 smithi116 podman[71673]: 2024-04-26 04:58:10.663245374 +0000 UTC m=+0.373850581 container start 8dbc41610354acb93cffbfb057018bfdf217bf2e63298919209d97511c696720 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:58:10.992 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:10 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[71688]: ts=2024-04-26T04:58:10.686Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-26T04:58:10.992 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:10 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[71688]: ts=2024-04-26T04:58:10.686Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-26T04:58:10.992 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:10 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[71688]: ts=2024-04-26T04:58:10.686Z caller=main.go:570 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi116 (none))" 2024-04-26T04:58:10.993 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:10 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[71688]: ts=2024-04-26T04:58:10.686Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-26T04:58:10.993 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:10 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[71688]: ts=2024-04-26T04:58:10.686Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-26T04:58:10.993 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:10 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[71688]: ts=2024-04-26T04:58:10.687Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-26T04:58:10.993 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:10 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[71688]: ts=2024-04-26T04:58:10.688Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-26T04:58:10.993 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:10 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[71688]: ts=2024-04-26T04:58:10.689Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-26T04:58:10.993 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:10 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[71688]: ts=2024-04-26T04:58:10.689Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-26T04:58:10.993 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:10 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[71688]: ts=2024-04-26T04:58:10.691Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-26T04:58:10.993 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:10 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[71688]: ts=2024-04-26T04:58:10.691Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.173µs 2024-04-26T04:58:10.993 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:10 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[71688]: ts=2024-04-26T04:58:10.691Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-26T04:58:10.993 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:10 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[71688]: ts=2024-04-26T04:58:10.691Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-04-26T04:58:10.993 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:10 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[71688]: ts=2024-04-26T04:58:10.704Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-04-26T04:58:10.993 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:10 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[71688]: ts=2024-04-26T04:58:10.704Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-04-26T04:58:10.993 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:10 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[71688]: ts=2024-04-26T04:58:10.704Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=20.511µs wal_replay_duration=13.362742ms wbl_replay_duration=99ns total_replay_duration=13.401974ms 2024-04-26T04:58:10.994 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:10 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[71688]: ts=2024-04-26T04:58:10.707Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-26T04:58:10.994 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:10 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[71688]: ts=2024-04-26T04:58:10.707Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-26T04:58:10.994 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:10 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[71688]: ts=2024-04-26T04:58:10.707Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-26T04:58:10.994 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:10 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[71688]: ts=2024-04-26T04:58:10.725Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=17.799531ms db_storage=775ns remote_storage=1.138µs web_handler=374ns query_engine=621ns scrape=379.501µs scrape_sd=95.422µs notify=10.077µs notify_sd=11.212µs rules=17.080265ms tracing=6.812µs 2024-04-26T04:58:10.994 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:10 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[71688]: ts=2024-04-26T04:58:10.725Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-26T04:58:10.994 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:10 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[71688]: ts=2024-04-26T04:58:10.725Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-26T04:58:10.994 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:10 smithi116 bash[71673]: 8dbc41610354acb93cffbfb057018bfdf217bf2e63298919209d97511c696720 2024-04-26T04:58:10.994 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:10 smithi116 systemd[1]: Started Ceph prometheus.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:58:11.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:11 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/2754695183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:11.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:11 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:11.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:11 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:11.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:11 smithi116 ceph-mon[29897]: Reconfiguring grafana.a (dependencies changed)... 2024-04-26T04:58:11.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:11 smithi116 ceph-mon[29897]: Reconfiguring daemon grafana.a on smithi116 2024-04-26T04:58:11.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:11 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/2754695183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:11.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:11 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:11.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:11 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:11.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:11 smithi012 ceph-mon[26617]: Reconfiguring grafana.a (dependencies changed)... 2024-04-26T04:58:11.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:11 smithi012 ceph-mon[26617]: Reconfiguring daemon grafana.a on smithi116 2024-04-26T04:58:11.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:11 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/2754695183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:11.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:11 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:11.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:11 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:11.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:11 smithi012 ceph-mon[37567]: Reconfiguring grafana.a (dependencies changed)... 2024-04-26T04:58:11.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:11 smithi012 ceph-mon[37567]: Reconfiguring daemon grafana.a on smithi116 2024-04-26T04:58:13.159 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:12 smithi012 ceph-mon[26617]: pgmap v21: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:13.160 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:12 smithi012 ceph-mon[37567]: pgmap v21: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:13.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:12 smithi116 ceph-mon[29897]: pgmap v21: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:13.512 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 04:58:13 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[81716]: ts=2024-04-26T04:58:13.159Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003362701s 2024-04-26T04:58:14.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:13 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/1454627983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:14.262 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:13 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/1454627983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:14.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:13 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1454627983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:15.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:14 smithi012 ceph-mon[26617]: pgmap v22: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:15.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:14 smithi012 ceph-mon[37567]: pgmap v22: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:15.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:14 smithi116 ceph-mon[29897]: pgmap v22: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:16.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:15 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/134420791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:16.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:15 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/134420791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:16.262 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:15 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/134420791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:17.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:16 smithi116 ceph-mon[29897]: pgmap v23: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:17.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:16 smithi012 ceph-mon[26617]: pgmap v23: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:17.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:16 smithi012 ceph-mon[37567]: pgmap v23: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:19.110 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:18 smithi116 ceph-mon[29897]: pgmap v24: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:19.110 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:18 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/882071461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:19.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:18 smithi012 ceph-mon[26617]: pgmap v24: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:19.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:18 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/882071461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:19.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:18 smithi012 ceph-mon[37567]: pgmap v24: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:19.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:18 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/882071461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:20.241 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:20 smithi116 systemd[1]: Stopping Ceph grafana.a for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:58:20.741 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:20 smithi116 podman[71996]: 2024-04-26 04:58:20.431687782 +0000 UTC m=+0.118267846 container died c7fd7ca82930f4afcf7e2c48112817b9cdfcc301586b96887e7e6f19b48111c6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git, architecture=x86_64, vendor=Red Hat, Inc., distribution-scope=public, io.buildah.version=1.32.0, io.openshift.expose-services=, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, maintainer=Paul Cuzner , description=Ceph Grafana Container) 2024-04-26T04:58:21.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:20 smithi116 ceph-mon[29897]: pgmap v25: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:21.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:20 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/3535652541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:21.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:20 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:21.242 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:20 smithi116 podman[71996]: 2024-04-26 04:58:20.854921486 +0000 UTC m=+0.541501554 container cleanup c7fd7ca82930f4afcf7e2c48112817b9cdfcc301586b96887e7e6f19b48111c6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, io.buildah.version=1.32.0, architecture=x86_64, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git, io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , name=ubi8, description=Ceph Grafana Container, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, build-date=2022-04-29T11:48:11.055897) 2024-04-26T04:58:21.242 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:20 smithi116 bash[71996]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a 2024-04-26T04:58:21.242 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:21 smithi116 podman[72010]: 2024-04-26 04:58:21.031546543 +0000 UTC m=+0.602790688 container remove c7fd7ca82930f4afcf7e2c48112817b9cdfcc301586b96887e7e6f19b48111c6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, release=239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, com.redhat.component=ubi8-container, description=Ceph Grafana Container, io.buildah.version=1.32.0, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , vcs-type=git, distribution-scope=public, io.openshift.tags=base rhel8, name=ubi8) 2024-04-26T04:58:21.242 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:21 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-04-26T04:58:21.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:20 smithi012 ceph-mon[26617]: pgmap v25: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:21.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:20 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/3535652541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:21.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:20 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:21.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:20 smithi012 ceph-mon[37567]: pgmap v25: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:21.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:20 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/3535652541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:21.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:20 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:21.626 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:21 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:58:21.626 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:21 smithi116 systemd[1]: Stopped Ceph grafana.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:58:21.626 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:21 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@grafana.a.service: Consumed 5.125s CPU time. 2024-04-26T04:58:21.626 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:21 smithi116 systemd[1]: Starting Ceph grafana.a for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:58:21.626 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:21 smithi116 podman[72103]: 2024-04-26 04:58:21.383422327 +0000 UTC m=+0.016999311 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:58:21.626 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:21 smithi116 podman[72103]: 2024-04-26 04:58:21.499088845 +0000 UTC m=+0.132665802 container create edf547f9e03ee7aa664a5972af0fd4c7c7b8a0cc89ae4e03b09f894d76099ef7 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, distribution-scope=public, io.openshift.tags=base rhel8, vcs-type=git, name=ubi8, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8) 2024-04-26T04:58:21.924 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:21 smithi116 podman[72103]: 2024-04-26 04:58:21.768413587 +0000 UTC m=+0.401990550 container init edf547f9e03ee7aa664a5972af0fd4c7c7b8a0cc89ae4e03b09f894d76099ef7 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, distribution-scope=public, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , io.openshift.expose-services=, name=ubi8, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, description=Ceph Grafana Container, com.redhat.component=ubi8-container, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-04-26T04:58:21.924 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:21 smithi116 podman[72103]: 2024-04-26 04:58:21.77094622 +0000 UTC m=+0.404523181 container start edf547f9e03ee7aa664a5972af0fd4c7c7b8a0cc89ae4e03b09f894d76099ef7 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , description=Ceph Grafana Container, version=8.5, vendor=Red Hat, Inc., architecture=x86_64, io.openshift.expose-services=, release=239.1651231664, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, build-date=2022-04-29T11:48:11.055897, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.component=ubi8-container) 2024-04-26T04:58:22.241 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[72118]: logger=settings t=2024-04-26T04:58:21.924356105Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:58:22.241 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:21 smithi116 bash[72103]: edf547f9e03ee7aa664a5972af0fd4c7c7b8a0cc89ae4e03b09f894d76099ef7 2024-04-26T04:58:22.241 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:22 smithi116 systemd[1]: Started Ceph grafana.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:58:22.241 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:22 smithi116 podman[72135]: 2024-04-26 04:58:22.029236699 +0000 UTC m=+0.097693307 container died edf547f9e03ee7aa664a5972af0fd4c7c7b8a0cc89ae4e03b09f894d76099ef7 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, com.redhat.component=ubi8-container, name=ubi8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, vcs-type=git, io.openshift.expose-services=, build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64, description=Ceph Grafana Container, io.openshift.tags=base rhel8, release=239.1651231664, maintainer=Paul Cuzner , version=8.5) 2024-04-26T04:58:23.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:23 smithi116 ceph-mon[29897]: pgmap v26: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:23.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:23 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:23.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:23 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:23.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:23 smithi116 ceph-mon[29897]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-26T04:58:23.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:23 smithi116 ceph-mon[29897]: Reconfiguring daemon node-exporter.b on smithi116 2024-04-26T04:58:23.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:23 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/3737277190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:23.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:23 smithi012 ceph-mon[26617]: pgmap v26: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:23.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:23 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:23.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:23 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:23.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:23 smithi012 ceph-mon[26617]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-26T04:58:23.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:23 smithi012 ceph-mon[26617]: Reconfiguring daemon node-exporter.b on smithi116 2024-04-26T04:58:23.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:23 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/3737277190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:23.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:23 smithi012 ceph-mon[37567]: pgmap v26: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:23.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:23 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:23.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:23 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:23.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:23 smithi012 ceph-mon[37567]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-26T04:58:23.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:23 smithi012 ceph-mon[37567]: Reconfiguring daemon node-exporter.b on smithi116 2024-04-26T04:58:23.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:23 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/3737277190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:24.645 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:24 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:58:24] "GET /metrics HTTP/1.1" 200 33707 "" "Prometheus/2.43.0" 2024-04-26T04:58:24.927 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:24 smithi116 systemd[1]: Stopping Ceph node-exporter.b for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:58:24.927 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:24 smithi116 podman[72135]: 2024-04-26 04:58:24.645498 +0000 UTC m=+2.713954640 container remove edf547f9e03ee7aa664a5972af0fd4c7c7b8a0cc89ae4e03b09f894d76099ef7 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64, distribution-scope=public, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, description=Ceph Grafana Container, name=ubi8, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, maintainer=Paul Cuzner , version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0) 2024-04-26T04:58:24.927 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:24 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:58:25.241 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:24 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:58:25.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:25 smithi116 ceph-mon[29897]: pgmap v27: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:25.242 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:25 smithi116 podman[72296]: 2024-04-26 04:58:25.075348943 +0000 UTC m=+0.106824054 container died da87ef4e90ea0c2a7856ff12c1f97cf903fd1151c791917236dce03e6e9cb1fa (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:58:25.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:25 smithi012 ceph-mon[26617]: pgmap v27: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:25.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:25 smithi012 ceph-mon[37567]: pgmap v27: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:25.708 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:25 smithi116 podman[72296]: 2024-04-26 04:58:25.415354536 +0000 UTC m=+0.446829645 container cleanup da87ef4e90ea0c2a7856ff12c1f97cf903fd1151c791917236dce03e6e9cb1fa (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:58:25.708 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:25 smithi116 bash[72296]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b 2024-04-26T04:58:25.991 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:25 smithi116 podman[72310]: 2024-04-26 04:58:25.708233815 +0000 UTC m=+0.637479826 container remove da87ef4e90ea0c2a7856ff12c1f97cf903fd1151c791917236dce03e6e9cb1fa (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:58:25.992 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:25 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-26T04:58:25.992 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:25 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-26T04:58:25.992 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:25 smithi116 systemd[1]: Stopped Ceph node-exporter.b for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:58:25.992 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:25 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@node-exporter.b.service: Consumed 2.024s CPU time. 2024-04-26T04:58:25.992 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:25 smithi116 systemd[1]: Starting Ceph node-exporter.b for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:58:26.353 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/1176999610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:26.354 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 podman[72407]: 2024-04-26 04:58:25.946275643 +0000 UTC m=+0.017673699 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-26T04:58:26.354 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 podman[72407]: 2024-04-26 04:58:26.067674503 +0000 UTC m=+0.139072530 container create 7d0b1e809531c495421cfadd77ef13b3c561449f6a88f0fb881426fb653c5704 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:58:26.354 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 podman[72407]: 2024-04-26 04:58:26.269920123 +0000 UTC m=+0.341318151 container init 7d0b1e809531c495421cfadd77ef13b3c561449f6a88f0fb881426fb653c5704 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:58:26.354 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 podman[72407]: 2024-04-26 04:58:26.272401638 +0000 UTC m=+0.343799668 container start 7d0b1e809531c495421cfadd77ef13b3c561449f6a88f0fb881426fb653c5704 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:58:26.354 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.275Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-26T04:58:26.355 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.275Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-26T04:58:26.355 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-26T04:58:26.355 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-26T04:58:26.355 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-26T04:58:26.355 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-26T04:58:26.355 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-26T04:58:26.355 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=arp 2024-04-26T04:58:26.355 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-26T04:58:26.355 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-26T04:58:26.355 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-26T04:58:26.355 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-26T04:58:26.355 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-26T04:58:26.355 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-26T04:58:26.356 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-26T04:58:26.356 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-26T04:58:26.356 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=edac 2024-04-26T04:58:26.356 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-26T04:58:26.356 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-26T04:58:26.356 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-26T04:58:26.356 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-26T04:58:26.356 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-26T04:58:26.356 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-26T04:58:26.356 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-26T04:58:26.356 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-26T04:58:26.356 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-26T04:58:26.357 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-26T04:58:26.357 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-26T04:58:26.357 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-26T04:58:26.357 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-26T04:58:26.357 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-26T04:58:26.357 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-26T04:58:26.357 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-26T04:58:26.357 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=os 2024-04-26T04:58:26.357 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-26T04:58:26.357 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-26T04:58:26.357 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-26T04:58:26.357 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-26T04:58:26.357 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-26T04:58:26.358 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-26T04:58:26.358 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-26T04:58:26.358 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=stat 2024-04-26T04:58:26.358 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-26T04:58:26.358 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-26T04:58:26.358 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-26T04:58:26.358 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=time 2024-04-26T04:58:26.358 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-26T04:58:26.358 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=uname 2024-04-26T04:58:26.358 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-26T04:58:26.358 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-26T04:58:26.358 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.276Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-26T04:58:26.358 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.277Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-26T04:58:26.359 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b[72421]: ts=2024-04-26T04:58:26.277Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-26T04:58:26.359 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 bash[72407]: 7d0b1e809531c495421cfadd77ef13b3c561449f6a88f0fb881426fb653c5704 2024-04-26T04:58:26.512 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:26 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1176999610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:26.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:26 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/1176999610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:26.674 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 04:58:26 smithi116 systemd[1]: Started Ceph node-exporter.b for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:58:26.675 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: [26/Apr/2024:04:58:26] ENGINE Bus STOPPING 2024-04-26T04:58:26.675 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: [26/Apr/2024:04:58:26] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:58:26.675 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: [26/Apr/2024:04:58:26] ENGINE Bus STOPPED 2024-04-26T04:58:26.675 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: [26/Apr/2024:04:58:26] ENGINE Bus STARTING 2024-04-26T04:58:26.675 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: [26/Apr/2024:04:58:26] ENGINE Serving on http://:::9283 2024-04-26T04:58:26.675 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: [26/Apr/2024:04:58:26] ENGINE Bus STARTED 2024-04-26T04:58:26.675 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: [26/Apr/2024:04:58:26] ENGINE Bus STOPPING 2024-04-26T04:58:26.675 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: [26/Apr/2024:04:58:26] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:58:26.675 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: [26/Apr/2024:04:58:26] ENGINE Bus STOPPED 2024-04-26T04:58:26.675 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: [26/Apr/2024:04:58:26] ENGINE Bus STARTING 2024-04-26T04:58:26.676 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: [26/Apr/2024:04:58:26] ENGINE Serving on http://:::9283 2024-04-26T04:58:26.676 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: [26/Apr/2024:04:58:26] ENGINE Bus STARTED 2024-04-26T04:58:26.991 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: [26/Apr/2024:04:58:26] ENGINE Bus STOPPING 2024-04-26T04:58:26.992 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: [26/Apr/2024:04:58:26] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:58:26.992 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: [26/Apr/2024:04:58:26] ENGINE Bus STOPPED 2024-04-26T04:58:26.992 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: [26/Apr/2024:04:58:26] ENGINE Bus STARTING 2024-04-26T04:58:26.992 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: [26/Apr/2024:04:58:26] ENGINE Serving on http://:::9283 2024-04-26T04:58:26.992 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: [26/Apr/2024:04:58:26] ENGINE Bus STARTED 2024-04-26T04:58:27.298 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:27 smithi012 ceph-mon[37567]: pgmap v28: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:27.299 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:27 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:27.299 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:27 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:27.299 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:27 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:58:27.299 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:27 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi012.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T04:58:27.299 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:27 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:27.299 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:27 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:58:27.299 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:27 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi116:9095"}]: dispatch 2024-04-26T04:58:27.299 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:27 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:27.299 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:27 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:58:27.299 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:27 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi116:3000"}]: dispatch 2024-04-26T04:58:27.299 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:27 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:27.299 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:27 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:58:27.299 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:27 smithi012 ceph-mon[26617]: pgmap v28: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:27.300 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:27 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:27.300 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:27 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:27.300 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:27 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:58:27.300 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:27 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi012.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T04:58:27.300 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:27 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:27.300 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:27 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:58:27.300 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:27 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi116:9095"}]: dispatch 2024-04-26T04:58:27.300 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:27 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:27.300 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:27 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:58:27.300 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:27 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi116:3000"}]: dispatch 2024-04-26T04:58:27.300 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:27 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:27.300 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:27 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:58:27.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:27 smithi116 ceph-mon[29897]: pgmap v28: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:27.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:27 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:27.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:27 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:27.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:27 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:58:27.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:27 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi012.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T04:58:27.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:27 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:27.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:27 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:58:27.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:27 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi116:9095"}]: dispatch 2024-04-26T04:58:27.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:27 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:27.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:27 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:58:27.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:27 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi116:3000"}]: dispatch 2024-04-26T04:58:27.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:27 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:27.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:27 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:58:28.262 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:28 smithi012 ceph-mon[37567]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:58:28.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:28 smithi012 ceph-mon[37567]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi012.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T04:58:28.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:28 smithi012 ceph-mon[37567]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:58:28.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:28 smithi012 ceph-mon[37567]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi116:9095"}]: dispatch 2024-04-26T04:58:28.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:28 smithi012 ceph-mon[37567]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:58:28.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:28 smithi012 ceph-mon[37567]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi116:3000"}]: dispatch 2024-04-26T04:58:28.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:28 smithi012 ceph-mon[37567]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-26T04:58:28.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:28 smithi012 ceph-mon[37567]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-26T04:58:28.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:28 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/2226480513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:28.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:28 smithi012 ceph-mon[26617]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:58:28.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:28 smithi012 ceph-mon[26617]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi012.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T04:58:28.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:28 smithi012 ceph-mon[26617]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:58:28.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:28 smithi012 ceph-mon[26617]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi116:9095"}]: dispatch 2024-04-26T04:58:28.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:28 smithi012 ceph-mon[26617]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:58:28.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:28 smithi012 ceph-mon[26617]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi116:3000"}]: dispatch 2024-04-26T04:58:28.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:28 smithi012 ceph-mon[26617]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-26T04:58:28.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:28 smithi012 ceph-mon[26617]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-26T04:58:28.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:28 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/2226480513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:28.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:28 smithi116 ceph-mon[29897]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:58:28.550 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:28 smithi116 ceph-mon[29897]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi012.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T04:58:28.550 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:28 smithi116 ceph-mon[29897]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:58:28.550 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:28 smithi116 ceph-mon[29897]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi116:9095"}]: dispatch 2024-04-26T04:58:28.550 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:28 smithi116 ceph-mon[29897]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:58:28.550 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:28 smithi116 ceph-mon[29897]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi116:3000"}]: dispatch 2024-04-26T04:58:28.550 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:28 smithi116 ceph-mon[29897]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-26T04:58:28.550 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:28 smithi116 ceph-mon[29897]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-26T04:58:28.550 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:28 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/2226480513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:28.991 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:28 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[71688]: ts=2024-04-26T04:58:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi116:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi116.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:58:29.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:29 smithi012 ceph-mon[26617]: pgmap v29: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:29.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:29 smithi012 ceph-mon[26617]: Upgrade: Updating mgr.y 2024-04-26T04:58:29.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:29 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:29.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:29 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:58:29.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:29 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:58:29.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:29 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:58:29.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:29 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:58:29.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:29 smithi012 ceph-mon[26617]: Deploying daemon mgr.y on smithi012 2024-04-26T04:58:29.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:29 smithi012 ceph-mon[37567]: pgmap v29: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:29.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:29 smithi012 ceph-mon[37567]: Upgrade: Updating mgr.y 2024-04-26T04:58:29.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:29 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:29.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:29 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:58:29.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:29 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:58:29.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:29 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:58:29.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:29 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:58:29.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:29 smithi012 ceph-mon[37567]: Deploying daemon mgr.y on smithi012 2024-04-26T04:58:29.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:29 smithi116 ceph-mon[29897]: pgmap v29: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:29.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:29 smithi116 ceph-mon[29897]: Upgrade: Updating mgr.y 2024-04-26T04:58:29.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:29 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:29.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:29 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:58:29.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:29 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:58:29.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:29 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:58:29.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:29 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:58:29.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:29 smithi116 ceph-mon[29897]: Deploying daemon mgr.y on smithi012 2024-04-26T04:58:30.013 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:29 smithi012 systemd[1]: Stopping Ceph mgr.y for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:58:30.424 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:30 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1357500652' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:30.424 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:30 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/1357500652' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:30.424 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:30 smithi012 podman[82573]: 2024-04-26 04:58:30.170561431 +0000 UTC m=+0.136317446 container died cdd1f8559c4e1666601bce8003da6568c80991d624c47d03e881ce11b2c1237f (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=quincy-c61472c, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-04-26T04:58:30.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:30 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/1357500652' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:30.700 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:30 smithi012 podman[82573]: 2024-04-26 04:58:30.514599026 +0000 UTC m=+0.480355043 container cleanup cdd1f8559c4e1666601bce8003da6568c80991d624c47d03e881ce11b2c1237f (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y, RELEASE=quincy-c61472c, ceph=True, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:58:30.700 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:30 smithi012 bash[82573]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y 2024-04-26T04:58:30.959 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:30 smithi012 podman[82588]: 2024-04-26 04:58:30.699763599 +0000 UTC m=+0.527380303 container remove cdd1f8559c4e1666601bce8003da6568c80991d624c47d03e881ce11b2c1237f (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=quincy-c61472c, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-26T04:58:30.959 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:30 smithi012 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-04-26T04:58:30.959 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:30 smithi012 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-04-26T04:58:30.959 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:30 smithi012 systemd[1]: Stopped Ceph mgr.y for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:58:30.959 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:30 smithi012 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mgr.y.service: Consumed 59.499s CPU time. 2024-04-26T04:58:31.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:31 smithi012 ceph-mon[26617]: pgmap v30: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:31.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:31 smithi012 ceph-mon[37567]: pgmap v30: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:31.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:31 smithi012 systemd[1]: Starting Ceph mgr.y for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:58:31.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:31 smithi116 ceph-mon[29897]: pgmap v30: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:31.640 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:31 smithi012 podman[82718]: 2024-04-26 04:58:31.270570344 +0000 UTC m=+0.018808008 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:58:31.640 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:31 smithi012 podman[82718]: 2024-04-26 04:58:31.375914382 +0000 UTC m=+0.124152025 container create 3a46d00317727c28523c6b1f8b84b372e3e8f8f1382fb10ef9d709d4703b0716 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-04-26T04:58:31.640 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:31 smithi012 podman[82718]: 2024-04-26 04:58:31.611763518 +0000 UTC m=+0.360001163 container init 3a46d00317727c28523c6b1f8b84b372e3e8f8f1382fb10ef9d709d4703b0716 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-04-26T04:58:31.640 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:31 smithi012 podman[82718]: 2024-04-26 04:58:31.614728215 +0000 UTC m=+0.362965868 container start 3a46d00317727c28523c6b1f8b84b372e3e8f8f1382fb10ef9d709d4703b0716 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, ceph=True, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-04-26T04:58:31.640 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:31 smithi012 ceph-mgr[82737]: -- 172.21.15.12:0/2723716882 <== mon.2 v2:172.21.15.12:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x559e4c07c5a0 con 0x559e4c051000 2024-04-26T04:58:31.972 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:31 smithi012 bash[82718]: 3a46d00317727c28523c6b1f8b84b372e3e8f8f1382fb10ef9d709d4703b0716 2024-04-26T04:58:31.972 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:31 smithi012 systemd[1]: Started Ceph mgr.y for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:58:31.972 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:31 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:58:31.771+0000 7f31d61be200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:58:31.972 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:31 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:58:31.971+0000 7f31d61be200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:58:32.262 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:32 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:58:32.084+0000 7f31d61be200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:58:32.762 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:32 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:58:32.448+0000 7f31d61be200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:58:32.763 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:32 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:58:32.540+0000 7f31d61be200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:58:32.840 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:32 smithi116 ceph-mon[29897]: pgmap v31: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:32.840 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:32 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:32.840 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:32 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:32.840 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:32 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:58:32.840 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:32 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/27591679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:33.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:32 smithi012 ceph-mon[26617]: pgmap v31: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:33.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:32 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:33.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:32 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:33.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:32 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:58:33.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:32 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/27591679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:33.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:32 smithi012 ceph-mon[37567]: pgmap v31: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:33.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:32 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:33.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:32 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:33.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:32 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:58:33.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:32 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/27591679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:34.262 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:33 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:58:33.891+0000 7f31d61be200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:58:34.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:34 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:58:34.112+0000 7f31d61be200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:58:34.564 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:34 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:58:34.294+0000 7f31d61be200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:58:34.741 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:34 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:58:34] "GET /metrics HTTP/1.1" 200 33707 "" "Prometheus/2.43.0" 2024-04-26T04:58:34.939 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:34 smithi012 ceph-mon[26617]: pgmap v32: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:34.939 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:34 smithi012 ceph-mon[37567]: pgmap v32: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:34.939 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:34 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:58:34.612+0000 7f31d61be200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:58:34.939 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:34 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:58:34.710+0000 7f31d61be200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:58:35.092 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:34 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 1. 2024-04-26T04:58:35.092 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:34 smithi116 systemd[1]: Stopped Ceph grafana.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:58:35.092 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:35 smithi116 systemd[1]: Starting Ceph grafana.a for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:58:35.092 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:34 smithi116 ceph-mon[29897]: pgmap v32: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:35.249 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:34 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:58:34.938+0000 7f31d61be200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:58:35.249 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:35 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:58:35.146+0000 7f31d61be200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:58:36.038 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:35 smithi116 podman[73033]: 2024-04-26 04:58:35.643412634 +0000 UTC m=+0.017711401 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:58:36.039 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:35 smithi116 podman[73033]: 2024-04-26 04:58:35.777512354 +0000 UTC m=+0.151811095 container create 18abc3f1f37bf8716dce7297f1739e1338b8cb26bb3affddc910fbb4838799f1 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, vcs-type=git, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.component=ubi8-container, description=Ceph Grafana Container, distribution-scope=public, release=239.1651231664, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, vendor=Red Hat, Inc.) 2024-04-26T04:58:36.039 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:35 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/957120927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:36.039 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:35 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:36.039 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:35 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:36.039 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:35 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:36.039 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:35 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:36.039 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:35 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:36.262 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:35 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:58:35.823+0000 7f31d61be200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:58:36.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:35 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/957120927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:36.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:35 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:36.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:35 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:36.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:35 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:36.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:35 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:36.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:35 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:36.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:35 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/957120927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:36.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:35 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:36.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:35 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:36.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:35 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:36.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:35 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:36.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:35 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:36.492 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:36 smithi116 podman[73033]: 2024-04-26 04:58:36.038588141 +0000 UTC m=+0.412886882 container init 18abc3f1f37bf8716dce7297f1739e1338b8cb26bb3affddc910fbb4838799f1 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, vendor=Red Hat, Inc., vcs-type=git, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, name=ubi8, version=8.5, architecture=x86_64, description=Ceph Grafana Container, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, maintainer=Paul Cuzner , io.buildah.version=1.32.0) 2024-04-26T04:58:36.492 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:36 smithi116 podman[73033]: 2024-04-26 04:58:36.041591349 +0000 UTC m=+0.415890102 container start 18abc3f1f37bf8716dce7297f1739e1338b8cb26bb3affddc910fbb4838799f1 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, vcs-type=git, release=239.1651231664, maintainer=Paul Cuzner , io.buildah.version=1.32.0, io.openshift.expose-services=, build-date=2022-04-29T11:48:11.055897, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, version=8.5, vendor=Red Hat, Inc., architecture=x86_64, distribution-scope=public, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container) 2024-04-26T04:58:36.492 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:36 smithi116 bash[73033]: 18abc3f1f37bf8716dce7297f1739e1338b8cb26bb3affddc910fbb4838799f1 2024-04-26T04:58:36.492 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:36 smithi116 systemd[1]: Started Ceph grafana.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:58:36.492 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:36 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[73047]: logger=settings t=2024-04-26T04:58:36.191206992Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:58:36.492 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:36 smithi116 podman[73064]: 2024-04-26 04:58:36.223276254 +0000 UTC m=+0.020238446 container died 18abc3f1f37bf8716dce7297f1739e1338b8cb26bb3affddc910fbb4838799f1 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, com.redhat.component=ubi8-container, maintainer=Paul Cuzner , vcs-type=git, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., version=8.5, architecture=x86_64, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-04-26T04:58:36.616 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:36 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:58:36.287+0000 7f31d61be200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:58:36.616 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:36 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:58:36.393+0000 7f31d61be200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:58:36.616 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:36 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:58:36.615+0000 7f31d61be200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:58:37.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:36 smithi012 ceph-mon[26617]: pgmap v33: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:37.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:36 smithi012 ceph-mon[37567]: pgmap v33: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:37.116 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:36 smithi116 ceph-mon[29897]: pgmap v33: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:37.116 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:36 smithi116 podman[73064]: 2024-04-26 04:58:36.893379478 +0000 UTC m=+0.690341683 container remove 18abc3f1f37bf8716dce7297f1739e1338b8cb26bb3affddc910fbb4838799f1 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, release=239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, io.openshift.expose-services=, vendor=Red Hat, Inc., io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, version=8.5, name=ubi8, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, architecture=x86_64, io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-26T04:58:37.116 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:36 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:58:37.487 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:37 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:58:37.189+0000 7f31d61be200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:58:37.487 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:37 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:58:37.295+0000 7f31d61be200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:58:37.491 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:37 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:58:37.762 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:37 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:58:37.567+0000 7f31d61be200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:58:37.763 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:37 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:58:37.714+0000 7f31d61be200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:58:38.045 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:37 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:38.045 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:37 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:38.045 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:37 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1585497158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:38.046 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:37 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:58:37.817+0000 7f31d61be200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:58:38.046 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:37 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:58:37.942+0000 7f31d61be200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:58:38.046 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:38 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:58:38.044+0000 7f31d61be200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:58:38.046 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:37 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:38.046 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:37 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:38.046 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:37 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/1585497158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:38.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:37 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:38.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:37 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:38.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:37 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/1585497158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:38.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:38 smithi116 ceph-mon[29897]: pgmap v34: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:38.992 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:38 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[71688]: ts=2024-04-26T04:58:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi116:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi116.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:58:39.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:38 smithi012 ceph-mon[26617]: pgmap v34: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:39.013 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:38 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:58:38.924+0000 7f31d61be200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:58:39.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:38 smithi012 ceph-mon[37567]: pgmap v34: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:39.273 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:39 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:58:39.164+0000 7f31d61be200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:58:39.273 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:39 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: [26/Apr/2024:04:58:39] ENGINE Bus STARTING 2024-04-26T04:58:39.274 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:39 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: CherryPy Checker: 2024-04-26T04:58:39.274 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:39 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: The Application mounted at '' has an empty config. 2024-04-26T04:58:39.274 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:39 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:58:39.677 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:39 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: [26/Apr/2024:04:58:39] ENGINE Serving on http://:::9283 2024-04-26T04:58:39.677 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:39 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: [26/Apr/2024:04:58:39] ENGINE Bus STARTED 2024-04-26T04:58:39.984 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:39 smithi012 ceph-mon[26617]: Standby manager daemon y restarted 2024-04-26T04:58:39.984 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:39 smithi012 ceph-mon[26617]: Standby manager daemon y started 2024-04-26T04:58:39.984 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:39 smithi012 ceph-mon[26617]: from='mgr.? 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T04:58:39.984 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:39 smithi012 ceph-mon[26617]: from='mgr.? 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:58:39.984 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:39 smithi012 ceph-mon[26617]: from='mgr.? 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T04:58:39.984 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:39 smithi012 ceph-mon[26617]: from='mgr.? 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:58:39.984 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:39 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/161161925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:39.984 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:39 smithi012 ceph-mon[37567]: Standby manager daemon y restarted 2024-04-26T04:58:39.984 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:39 smithi012 ceph-mon[37567]: Standby manager daemon y started 2024-04-26T04:58:39.985 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:39 smithi012 ceph-mon[37567]: from='mgr.? 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T04:58:39.985 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:39 smithi012 ceph-mon[37567]: from='mgr.? 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:58:39.985 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:39 smithi012 ceph-mon[37567]: from='mgr.? 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T04:58:39.985 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:39 smithi012 ceph-mon[37567]: from='mgr.? 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:58:39.985 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:39 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/161161925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:40.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:39 smithi116 ceph-mon[29897]: Standby manager daemon y restarted 2024-04-26T04:58:40.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:39 smithi116 ceph-mon[29897]: Standby manager daemon y started 2024-04-26T04:58:40.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:39 smithi116 ceph-mon[29897]: from='mgr.? 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T04:58:40.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:39 smithi116 ceph-mon[29897]: from='mgr.? 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:58:40.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:39 smithi116 ceph-mon[29897]: from='mgr.? 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T04:58:40.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:39 smithi116 ceph-mon[29897]: from='mgr.? 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:58:40.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:39 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/161161925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:40.612 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: [26/Apr/2024:04:58:40] ENGINE Bus STOPPING 2024-04-26T04:58:40.612 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: [26/Apr/2024:04:58:40] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:58:40.612 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: [26/Apr/2024:04:58:40] ENGINE Bus STOPPED 2024-04-26T04:58:40.843 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:58:40.580+0000 7f8c5e407700 -1 mgr handle_mgr_map I was active but no longer am 2024-04-26T04:58:40.843 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: ignoring --setuser ceph since I am not root 2024-04-26T04:58:40.843 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: ignoring --setgroup ceph since I am not root 2024-04-26T04:58:40.843 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mgr[63141]: -- 172.21.15.116:0/3794818875 <== mon.1 v2:172.21.15.116:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55aec550a5a0 con 0x55aec54dec00 2024-04-26T04:58:40.843 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:58:40.757+0000 7fc17ca1c200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:58:41.006 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: [26/Apr/2024:04:58:40] ENGINE Bus STARTING 2024-04-26T04:58:41.006 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:58:40.717+0000 7f31345aa700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:58:41.007 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:58:40.717+0000 7f31345aa700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:58:41.007 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:58:40.717+0000 7f31345aa700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:58:41.007 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:58:40.717+0000 7f31345aa700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:58:41.007 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:58:40.717+0000 7f31345aa700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:58:41.007 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:58:40.717+0000 7f31385b2700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:58:41.007 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:58:40.717+0000 7f31385b2700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:58:41.007 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:58:40.717+0000 7f31385b2700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:58:41.007 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:58:40.717+0000 7f31385b2700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:58:41.008 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:58:40.717+0000 7f31385b2700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:58:41.008 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: [26/Apr/2024:04:58:40] ENGINE Serving on http://:::9283 2024-04-26T04:58:41.008 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: [26/Apr/2024:04:58:40] ENGINE Bus STARTED 2024-04-26T04:58:41.008 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: pgmap v35: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:41.008 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: mgrmap e27: x(active, since 64s), standbys: y 2024-04-26T04:58:41.008 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:41.008 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:41.008 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:58:41.008 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:58:41.008 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:41.009 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:58:41.009 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-26T04:58:41.009 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-26T04:58:41.009 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: Activating manager daemon y 2024-04-26T04:58:41.009 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: osdmap e79: 8 total, 8 up, 8 in 2024-04-26T04:58:41.009 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: from='mgr.24742 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-26T04:58:41.009 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: mgrmap e28: y(active, starting, since 0.00565442s) 2024-04-26T04:58:41.009 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:58:41.009 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:58:41.009 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:58:41.009 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi012.fdqwmo"}]: dispatch 2024-04-26T04:58:41.009 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi116.kduyef"}]: dispatch 2024-04-26T04:58:41.009 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:58:41.009 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:58:41.010 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:58:41.010 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:58:41.010 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:58:41.010 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:58:41.010 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:58:41.010 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:58:41.010 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:58:41.010 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:58:41.010 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:58:41.010 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: pgmap v35: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:41.010 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: mgrmap e27: x(active, since 64s), standbys: y 2024-04-26T04:58:41.011 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:41.011 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:41.011 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:58:41.011 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:58:41.011 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:41.011 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:58:41.011 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-26T04:58:41.011 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-26T04:58:41.011 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: Activating manager daemon y 2024-04-26T04:58:41.011 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: osdmap e79: 8 total, 8 up, 8 in 2024-04-26T04:58:41.011 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: from='mgr.24742 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-26T04:58:41.012 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: mgrmap e28: y(active, starting, since 0.00565442s) 2024-04-26T04:58:41.012 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:58:41.012 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:58:41.012 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:58:41.012 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi012.fdqwmo"}]: dispatch 2024-04-26T04:58:41.012 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi116.kduyef"}]: dispatch 2024-04-26T04:58:41.012 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:58:41.012 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:58:41.012 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:58:41.012 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:58:41.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:58:41.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:58:41.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:58:41.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:58:41.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:58:41.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:58:41.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:58:41.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:58:41.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: Manager daemon y is now available 2024-04-26T04:58:41.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:58:41.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:41.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:58:41.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:58:41.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:58:41.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:58:41.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:58:41.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: Manager daemon y is now available 2024-04-26T04:58:41.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:58:41.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:41.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:58:41.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:58:41.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:58:41.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:40 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:58:41.241 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:58:40.945+0000 7fc17ca1c200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:58:41.241 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:41 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:58:41.048+0000 7fc17ca1c200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:58:41.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: pgmap v35: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:41.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: mgrmap e27: x(active, since 64s), standbys: y 2024-04-26T04:58:41.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:41.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:41.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:58:41.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:58:41.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' 2024-04-26T04:58:41.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:58:41.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: from='mgr.24742 172.21.15.116:0/3655823265' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-26T04:58:41.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-26T04:58:41.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: Activating manager daemon y 2024-04-26T04:58:41.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: osdmap e79: 8 total, 8 up, 8 in 2024-04-26T04:58:41.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: from='mgr.24742 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-26T04:58:41.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: mgrmap e28: y(active, starting, since 0.00565442s) 2024-04-26T04:58:41.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:58:41.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:58:41.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:58:41.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi012.fdqwmo"}]: dispatch 2024-04-26T04:58:41.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi116.kduyef"}]: dispatch 2024-04-26T04:58:41.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:58:41.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:58:41.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:58:41.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:58:41.244 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:58:41.244 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:58:41.244 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:58:41.244 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:58:41.244 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:58:41.244 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:58:41.244 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:58:41.244 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:58:41.244 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: Manager daemon y is now available 2024-04-26T04:58:41.244 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:58:41.244 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:41.244 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:58:41.244 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:58:41.245 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:58:41.245 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:40 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:58:41.741 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:41 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:58:41.366+0000 7fc17ca1c200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:58:41.741 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:41 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:58:41.455+0000 7fc17ca1c200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:58:42.170 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:41 smithi116 ceph-mon[29897]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:58:42.171 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:41 smithi116 ceph-mon[29897]: mgrmap e29: y(active, since 1.00839s) 2024-04-26T04:58:42.171 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:41 smithi116 ceph-mon[29897]: pgmap v3: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:42.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:41 smithi012 ceph-mon[26617]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:58:42.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:41 smithi012 ceph-mon[26617]: mgrmap e29: y(active, since 1.00839s) 2024-04-26T04:58:42.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:41 smithi012 ceph-mon[26617]: pgmap v3: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:42.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:41 smithi012 ceph-mon[37567]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:58:42.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:41 smithi012 ceph-mon[37567]: mgrmap e29: y(active, since 1.00839s) 2024-04-26T04:58:42.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:41 smithi012 ceph-mon[37567]: pgmap v3: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:42.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:42 smithi116 ceph-mon[29897]: [26/Apr/2024:04:58:41] ENGINE Bus STARTING 2024-04-26T04:58:42.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:42 smithi116 ceph-mon[29897]: [26/Apr/2024:04:58:41] ENGINE Serving on https://172.21.15.12:7150 2024-04-26T04:58:42.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:42 smithi116 ceph-mon[29897]: [26/Apr/2024:04:58:41] ENGINE Serving on http://172.21.15.12:8765 2024-04-26T04:58:42.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:42 smithi116 ceph-mon[29897]: [26/Apr/2024:04:58:41] ENGINE Bus STARTED 2024-04-26T04:58:42.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:42 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/1737427428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:42.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:42 smithi116 ceph-mon[29897]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T04:58:42.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:42 smithi116 ceph-mon[29897]: Cluster is now healthy 2024-04-26T04:58:42.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:42 smithi116 ceph-mon[29897]: pgmap v4: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:42.992 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:42 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:58:42.690+0000 7fc17ca1c200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:58:42.992 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:42 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:58:42.868+0000 7fc17ca1c200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:58:43.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:42 smithi012 ceph-mon[26617]: [26/Apr/2024:04:58:41] ENGINE Bus STARTING 2024-04-26T04:58:43.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:42 smithi012 ceph-mon[26617]: [26/Apr/2024:04:58:41] ENGINE Serving on https://172.21.15.12:7150 2024-04-26T04:58:43.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:42 smithi012 ceph-mon[26617]: [26/Apr/2024:04:58:41] ENGINE Serving on http://172.21.15.12:8765 2024-04-26T04:58:43.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:42 smithi012 ceph-mon[26617]: [26/Apr/2024:04:58:41] ENGINE Bus STARTED 2024-04-26T04:58:43.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:42 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1737427428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:43.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:42 smithi012 ceph-mon[26617]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T04:58:43.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:42 smithi012 ceph-mon[26617]: Cluster is now healthy 2024-04-26T04:58:43.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:42 smithi012 ceph-mon[26617]: pgmap v4: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:43.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:42 smithi012 ceph-mon[37567]: [26/Apr/2024:04:58:41] ENGINE Bus STARTING 2024-04-26T04:58:43.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:42 smithi012 ceph-mon[37567]: [26/Apr/2024:04:58:41] ENGINE Serving on https://172.21.15.12:7150 2024-04-26T04:58:43.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:42 smithi012 ceph-mon[37567]: [26/Apr/2024:04:58:41] ENGINE Serving on http://172.21.15.12:8765 2024-04-26T04:58:43.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:42 smithi012 ceph-mon[37567]: [26/Apr/2024:04:58:41] ENGINE Bus STARTED 2024-04-26T04:58:43.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:42 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/1737427428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:43.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:42 smithi012 ceph-mon[37567]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T04:58:43.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:42 smithi012 ceph-mon[37567]: Cluster is now healthy 2024-04-26T04:58:43.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:42 smithi012 ceph-mon[37567]: pgmap v4: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:43.393 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:43 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:58:43.050+0000 7fc17ca1c200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:58:43.393 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:43 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:58:43.391+0000 7fc17ca1c200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:58:43.662 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:43 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:58:43.487+0000 7fc17ca1c200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:58:43.991 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:43 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:58:43.699+0000 7fc17ca1c200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:58:43.991 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:43 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:58:43.896+0000 7fc17ca1c200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:58:44.850 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:44 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:58:44.570+0000 7fc17ca1c200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:58:45.115 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:44 smithi116 ceph-mon[29897]: mgrmap e30: y(active, since 3s) 2024-04-26T04:58:45.115 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:44 smithi116 ceph-mon[29897]: pgmap v5: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:45.115 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:44 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/3512322401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:45.115 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:45 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:58:45.010+0000 7fc17ca1c200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:58:45.115 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:45 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:58:45.114+0000 7fc17ca1c200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:58:45.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:44 smithi012 ceph-mon[26617]: mgrmap e30: y(active, since 3s) 2024-04-26T04:58:45.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:44 smithi012 ceph-mon[26617]: pgmap v5: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:45.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:44 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/3512322401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:45.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:44 smithi012 ceph-mon[37567]: mgrmap e30: y(active, since 3s) 2024-04-26T04:58:45.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:44 smithi012 ceph-mon[37567]: pgmap v5: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:45.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:44 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/3512322401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:45.491 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:45 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:58:45.329+0000 7fc17ca1c200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:58:46.139 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:45 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:46.140 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:45 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:46.140 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:45 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:46.140 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:45 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:46.140 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:45 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:58:45.878+0000 7fc17ca1c200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:58:46.140 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:45 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:58:45.977+0000 7fc17ca1c200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:58:46.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:45 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:46.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:45 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:46.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:45 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:46.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:45 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:46.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:45 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:46.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:45 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:46.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:45 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:46.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:45 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:46.404 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:46 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:58:46.268+0000 7fc17ca1c200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:58:46.404 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:46 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:58:46.403+0000 7fc17ca1c200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:58:46.710 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:46 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:58:46.499+0000 7fc17ca1c200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:58:46.711 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:46 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:58:46.616+0000 7fc17ca1c200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:58:46.711 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:46 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:58:46.710+0000 7fc17ca1c200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:58:47.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:47 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:47.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:47 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:47.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:47 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:47.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:47 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:47.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:47 smithi012 ceph-mon[26617]: pgmap v6: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:47.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:47 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:47.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:47 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:47.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:47 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:47.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:47 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:47.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:47 smithi012 ceph-mon[37567]: pgmap v6: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:47.487 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:47 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:47.487 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:47 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:47.487 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:47 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:47.487 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:47 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:47.487 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:47 smithi116 ceph-mon[29897]: pgmap v6: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:58:47.487 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:47 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 2. 2024-04-26T04:58:47.487 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:47 smithi116 systemd[1]: Stopped Ceph grafana.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:58:47.487 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:47 smithi116 systemd[1]: Starting Ceph grafana.a for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:58:47.741 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:47 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:58:47.559+0000 7fc17ca1c200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:58:47.741 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:47 smithi116 podman[73971]: 2024-04-26 04:58:47.388374119 +0000 UTC m=+0.024974429 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:58:47.742 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:47 smithi116 podman[73971]: 2024-04-26 04:58:47.501440262 +0000 UTC m=+0.138040569 container create 90dab6fcb3eb1e06f2f55cc142b38990c56648d10317f9ac17453aedafad6ed7 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, description=Ceph Grafana Container, version=8.5, io.openshift.expose-services=, io.buildah.version=1.32.0, release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-04-26T04:58:48.024 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:47 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:58:47.805+0000 7fc17ca1c200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:58:48.024 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:47 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: [26/Apr/2024:04:58:47] ENGINE Bus STARTING 2024-04-26T04:58:48.024 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:47 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: CherryPy Checker: 2024-04-26T04:58:48.024 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:47 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: The Application mounted at '' has an empty config. 2024-04-26T04:58:48.024 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:47 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: 2024-04-26T04:58:48.025 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:47 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: [26/Apr/2024:04:58:47] ENGINE Serving on http://:::9283 2024-04-26T04:58:48.025 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:47 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: [26/Apr/2024:04:58:47] ENGINE Bus STARTED 2024-04-26T04:58:48.025 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:47 smithi116 podman[73971]: 2024-04-26 04:58:47.74533635 +0000 UTC m=+0.381936657 container init 90dab6fcb3eb1e06f2f55cc142b38990c56648d10317f9ac17453aedafad6ed7 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, release=239.1651231664, vcs-type=git, name=ubi8, io.openshift.tags=base rhel8, io.openshift.expose-services=, distribution-scope=public, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-04-29T11:48:11.055897) 2024-04-26T04:58:48.025 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:47 smithi116 podman[73971]: 2024-04-26 04:58:47.747980096 +0000 UTC m=+0.384580403 container start 90dab6fcb3eb1e06f2f55cc142b38990c56648d10317f9ac17453aedafad6ed7 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, architecture=x86_64, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, distribution-scope=public, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=239.1651231664, name=ubi8, io.openshift.tags=base rhel8, io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-04-26T04:58:48.025 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:47 smithi116 bash[73971]: 90dab6fcb3eb1e06f2f55cc142b38990c56648d10317f9ac17453aedafad6ed7 2024-04-26T04:58:48.025 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:47 smithi116 systemd[1]: Started Ceph grafana.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:58:48.025 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:47 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[73987]: logger=settings t=2024-04-26T04:58:47.94053016Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:58:48.025 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:47 smithi116 podman[74032]: 2024-04-26 04:58:47.966306781 +0000 UTC m=+0.018672176 container died 90dab6fcb3eb1e06f2f55cc142b38990c56648d10317f9ac17453aedafad6ed7 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, release=239.1651231664, description=Ceph Grafana Container, distribution-scope=public, io.buildah.version=1.32.0, version=8.5, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, build-date=2022-04-29T11:48:11.055897, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , architecture=x86_64, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-04-26T04:58:48.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:48 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/887481740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:48.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:48 smithi116 ceph-mon[29897]: Standby manager daemon x started 2024-04-26T04:58:48.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:48 smithi116 ceph-mon[29897]: from='mgr.? 172.21.15.116:0/3085352367' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:58:48.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:48 smithi116 ceph-mon[29897]: from='mgr.? 172.21.15.116:0/3085352367' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:58:48.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:48 smithi116 ceph-mon[29897]: from='mgr.? 172.21.15.116:0/3085352367' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:58:48.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:48 smithi116 ceph-mon[29897]: from='mgr.? 172.21.15.116:0/3085352367' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:58:48.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:48 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/887481740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:48.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:48 smithi012 ceph-mon[26617]: Standby manager daemon x started 2024-04-26T04:58:48.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:48 smithi012 ceph-mon[26617]: from='mgr.? 172.21.15.116:0/3085352367' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:58:48.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:48 smithi012 ceph-mon[26617]: from='mgr.? 172.21.15.116:0/3085352367' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:58:48.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:48 smithi012 ceph-mon[26617]: from='mgr.? 172.21.15.116:0/3085352367' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:58:48.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:48 smithi012 ceph-mon[26617]: from='mgr.? 172.21.15.116:0/3085352367' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:58:48.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:48 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/887481740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:48.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:48 smithi012 ceph-mon[37567]: Standby manager daemon x started 2024-04-26T04:58:48.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:48 smithi012 ceph-mon[37567]: from='mgr.? 172.21.15.116:0/3085352367' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:58:48.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:48 smithi012 ceph-mon[37567]: from='mgr.? 172.21.15.116:0/3085352367' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:58:48.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:48 smithi012 ceph-mon[37567]: from='mgr.? 172.21.15.116:0/3085352367' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:58:48.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:48 smithi012 ceph-mon[37567]: from='mgr.? 172.21.15.116:0/3085352367' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:58:48.842 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:48 smithi116 podman[74032]: 2024-04-26 04:58:48.624954513 +0000 UTC m=+0.677319892 container remove 90dab6fcb3eb1e06f2f55cc142b38990c56648d10317f9ac17453aedafad6ed7 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, distribution-scope=public, architecture=x86_64, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.32.0, vendor=Red Hat, Inc., release=239.1651231664, description=Ceph Grafana Container, io.openshift.tags=base rhel8) 2024-04-26T04:58:48.842 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:48 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:58:49.241 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:48 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:58:49.419 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:49 smithi012 ceph-mon[26617]: mgrmap e31: y(active, since 7s), standbys: x 2024-04-26T04:58:49.419 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:49 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:58:49.420 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:49 smithi012 ceph-mon[26617]: pgmap v7: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T04:58:49.420 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:49 smithi012 ceph-mon[37567]: mgrmap e31: y(active, since 7s), standbys: x 2024-04-26T04:58:49.420 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:49 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:58:49.420 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:49 smithi012 ceph-mon[37567]: pgmap v7: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T04:58:49.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:49 smithi116 ceph-mon[29897]: mgrmap e31: y(active, since 7s), standbys: x 2024-04-26T04:58:49.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:49 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:58:49.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:49 smithi116 ceph-mon[29897]: pgmap v7: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T04:58:50.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:50 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:50.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:50 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/1858914337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:50.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:50 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:50.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:50 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:58:50.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:50 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:58:50.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:50 smithi116 ceph-mon[29897]: Updating smithi012:/etc/ceph/ceph.conf 2024-04-26T04:58:50.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:50 smithi116 ceph-mon[29897]: Updating smithi116:/etc/ceph/ceph.conf 2024-04-26T04:58:50.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:50 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:50.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:50 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:50.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:50 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:50.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:50 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/1858914337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:50.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:50 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:50.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:50 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:58:50.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:50 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:58:50.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:50 smithi012 ceph-mon[26617]: Updating smithi012:/etc/ceph/ceph.conf 2024-04-26T04:58:50.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:50 smithi012 ceph-mon[26617]: Updating smithi116:/etc/ceph/ceph.conf 2024-04-26T04:58:50.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:50 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:50.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:50 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:50.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:50 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:50.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:50 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/1858914337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:50.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:50 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:50.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:50 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:58:50.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:50 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:58:50.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:50 smithi012 ceph-mon[37567]: Updating smithi012:/etc/ceph/ceph.conf 2024-04-26T04:58:50.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:50 smithi012 ceph-mon[37567]: Updating smithi116:/etc/ceph/ceph.conf 2024-04-26T04:58:50.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:50 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:50.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:50 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:51.850 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:51 smithi116 ceph-mon[29897]: Updating smithi012:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:58:51.850 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:51 smithi116 ceph-mon[29897]: Updating smithi116:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:58:51.850 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:51 smithi116 ceph-mon[29897]: Updating smithi116:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:58:51.850 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:51 smithi116 ceph-mon[29897]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:58:51.851 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:51 smithi116 ceph-mon[29897]: Updating smithi012:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:58:51.851 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:51 smithi116 ceph-mon[29897]: Updating smithi116:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:58:51.851 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:51 smithi116 ceph-mon[29897]: pgmap v8: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:58:51.851 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:51 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:51.851 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:51 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:51.851 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:51 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:51.851 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:51 smithi116 ceph-mon[29897]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:58:52.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:51 smithi012 ceph-mon[26617]: Updating smithi012:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:58:52.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:51 smithi012 ceph-mon[26617]: Updating smithi116:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:58:52.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:51 smithi012 ceph-mon[26617]: Updating smithi116:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:58:52.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:51 smithi012 ceph-mon[26617]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:58:52.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:51 smithi012 ceph-mon[26617]: Updating smithi012:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:58:52.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:51 smithi012 ceph-mon[26617]: Updating smithi116:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:58:52.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:51 smithi012 ceph-mon[26617]: pgmap v8: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:58:52.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:51 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:52.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:51 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:52.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:51 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:52.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:51 smithi012 ceph-mon[26617]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:58:52.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:51 smithi012 ceph-mon[37567]: Updating smithi012:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:58:52.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:51 smithi012 ceph-mon[37567]: Updating smithi116:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:58:52.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:51 smithi012 ceph-mon[37567]: Updating smithi116:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:58:52.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:51 smithi012 ceph-mon[37567]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:58:52.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:51 smithi012 ceph-mon[37567]: Updating smithi012:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:58:52.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:51 smithi012 ceph-mon[37567]: Updating smithi116:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:58:52.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:51 smithi012 ceph-mon[37567]: pgmap v8: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:58:52.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:51 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:52.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:51 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:52.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:51 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:52.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:51 smithi012 ceph-mon[37567]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:58:52.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:52 smithi116 ceph-mon[29897]: pgmap v9: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:58:52.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:52 smithi116 ceph-mon[29897]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:58:52.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:52 smithi116 ceph-mon[29897]: Reconfiguring daemon prometheus.a on smithi116 2024-04-26T04:58:52.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:52 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/559683215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:53.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:52 smithi012 ceph-mon[26617]: pgmap v9: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:58:53.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:52 smithi012 ceph-mon[26617]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:58:53.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:52 smithi012 ceph-mon[26617]: Reconfiguring daemon prometheus.a on smithi116 2024-04-26T04:58:53.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:52 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/559683215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:53.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:52 smithi012 ceph-mon[37567]: pgmap v9: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:58:53.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:52 smithi012 ceph-mon[37567]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:58:53.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:52 smithi012 ceph-mon[37567]: Reconfiguring daemon prometheus.a on smithi116 2024-04-26T04:58:53.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:52 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/559683215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:53.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:53 smithi116 ceph-mon[29897]: pgmap v10: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:58:54.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:53 smithi012 ceph-mon[26617]: pgmap v10: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:58:54.012 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:53 smithi012 ceph-mon[37567]: pgmap v10: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:58:54.704 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:54 smithi116 systemd[1]: Stopping Ceph prometheus.a for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:58:54.704 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:54 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[63136]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:58:54] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:58:54.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:54 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/2128919425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:54.992 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:54 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[71688]: ts=2024-04-26T04:58:54.752Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-26T04:58:54.992 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:54 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[71688]: ts=2024-04-26T04:58:54.752Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-26T04:58:54.992 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:54 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[71688]: ts=2024-04-26T04:58:54.752Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-26T04:58:54.992 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:54 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[71688]: ts=2024-04-26T04:58:54.752Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-26T04:58:54.992 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:54 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[71688]: ts=2024-04-26T04:58:54.752Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-26T04:58:54.992 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:54 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[71688]: ts=2024-04-26T04:58:54.752Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-26T04:58:54.992 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:54 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[71688]: ts=2024-04-26T04:58:54.752Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-26T04:58:54.992 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:54 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[71688]: ts=2024-04-26T04:58:54.752Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-26T04:58:54.992 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:54 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[71688]: ts=2024-04-26T04:58:54.752Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-26T04:58:54.992 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:54 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[71688]: ts=2024-04-26T04:58:54.752Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-26T04:58:54.992 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:54 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[71688]: ts=2024-04-26T04:58:54.752Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-26T04:58:54.992 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:54 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[71688]: ts=2024-04-26T04:58:54.752Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-26T04:58:54.992 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:54 smithi116 podman[75130]: 2024-04-26 04:58:54.762529502 +0000 UTC m=+0.112683538 container died 8dbc41610354acb93cffbfb057018bfdf217bf2e63298919209d97511c696720 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:58:55.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:54 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/2128919425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:55.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:54 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/2128919425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:55.403 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:55 smithi116 podman[75130]: 2024-04-26 04:58:55.143931051 +0000 UTC m=+0.494085092 container cleanup 8dbc41610354acb93cffbfb057018bfdf217bf2e63298919209d97511c696720 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:58:55.403 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:55 smithi116 bash[75130]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a 2024-04-26T04:58:55.704 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:55 smithi116 podman[75144]: 2024-04-26 04:58:55.403294783 +0000 UTC m=+0.640595862 container remove 8dbc41610354acb93cffbfb057018bfdf217bf2e63298919209d97511c696720 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:58:55.705 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:55 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-26T04:58:55.705 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:55 smithi116 systemd[1]: Stopped Ceph prometheus.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:58:55.705 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:55 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 1.123s CPU time. 2024-04-26T04:58:55.705 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:55 smithi116 systemd[1]: Starting Ceph prometheus.a for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:58:55.990 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:55 smithi116 podman[75242]: 2024-04-26 04:58:55.633441661 +0000 UTC m=+0.019381786 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-26T04:58:55.990 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:55 smithi116 podman[75242]: 2024-04-26 04:58:55.763163032 +0000 UTC m=+0.149103210 container create e9e5b93523ae1fbdc6028d790c32812e1be51d62d02337a9aece4bf3d215a4e5 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:58:55.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:55 smithi116 ceph-mon[29897]: pgmap v11: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:58:55.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:55 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:55.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:55 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:56.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:55 smithi012 ceph-mon[26617]: pgmap v11: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:58:56.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:55 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:56.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:55 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:56.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:55 smithi012 ceph-mon[37567]: pgmap v11: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:58:56.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:55 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:56.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:55 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:56.242 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:55 smithi116 podman[75242]: 2024-04-26 04:58:55.990208641 +0000 UTC m=+0.376148755 container init e9e5b93523ae1fbdc6028d790c32812e1be51d62d02337a9aece4bf3d215a4e5 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:58:56.242 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:55 smithi116 podman[75242]: 2024-04-26 04:58:55.992929538 +0000 UTC m=+0.378869656 container start e9e5b93523ae1fbdc6028d790c32812e1be51d62d02337a9aece4bf3d215a4e5 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:58:56.242 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:56 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T04:58:56.016Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-26T04:58:56.242 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:56 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T04:58:56.016Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-26T04:58:56.242 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:56 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T04:58:56.016Z caller=main.go:570 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi116 (none))" 2024-04-26T04:58:56.242 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:56 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T04:58:56.016Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-26T04:58:56.242 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:56 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T04:58:56.016Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-26T04:58:56.242 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:56 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T04:58:56.017Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-26T04:58:56.242 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:56 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T04:58:56.018Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-26T04:58:56.242 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:56 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T04:58:56.018Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-26T04:58:56.242 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:56 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T04:58:56.018Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-26T04:58:56.242 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:56 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T04:58:56.021Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-26T04:58:56.243 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:56 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T04:58:56.021Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.644µs 2024-04-26T04:58:56.243 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:56 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T04:58:56.021Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-26T04:58:56.243 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:56 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T04:58:56.021Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-04-26T04:58:56.243 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:56 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T04:58:56.035Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-04-26T04:58:56.243 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:56 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T04:58:56.037Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-04-26T04:58:56.243 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:56 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T04:58:56.037Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-04-26T04:58:56.243 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:56 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T04:58:56.037Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=25.698µs wal_replay_duration=16.363471ms wbl_replay_duration=120ns total_replay_duration=16.406755ms 2024-04-26T04:58:56.243 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:56 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T04:58:56.040Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-26T04:58:56.243 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:56 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T04:58:56.040Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-26T04:58:56.243 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:56 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T04:58:56.040Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-26T04:58:56.243 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:56 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T04:58:56.057Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=16.973448ms db_storage=806ns remote_storage=1.135µs web_handler=359ns query_engine=869ns scrape=213.198µs scrape_sd=121.673µs notify=25.54µs notify_sd=27.707µs rules=16.335957ms tracing=5.909µs 2024-04-26T04:58:56.243 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:56 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T04:58:56.057Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-26T04:58:56.243 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:56 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T04:58:56.057Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-26T04:58:56.243 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:56 smithi116 bash[75242]: e9e5b93523ae1fbdc6028d790c32812e1be51d62d02337a9aece4bf3d215a4e5 2024-04-26T04:58:56.243 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:58:56 smithi116 systemd[1]: Started Ceph prometheus.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:58:56.513 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:56 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: [26/Apr/2024:04:58:56] ENGINE Bus STOPPING 2024-04-26T04:58:56.513 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:56 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: [26/Apr/2024:04:58:56] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:58:56.513 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:56 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: [26/Apr/2024:04:58:56] ENGINE Bus STOPPED 2024-04-26T04:58:56.513 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:56 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: [26/Apr/2024:04:58:56] ENGINE Bus STARTING 2024-04-26T04:58:56.513 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:56 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: [26/Apr/2024:04:58:56] ENGINE Serving on http://:::9283 2024-04-26T04:58:56.514 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:58:56 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: [26/Apr/2024:04:58:56] ENGINE Bus STARTED 2024-04-26T04:58:57.413 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:57 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:57.414 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:57 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:57.414 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:57 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:58:57.414 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:57 smithi116 ceph-mon[29897]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:58:57.414 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:57 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi116.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:58:57.414 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:57 smithi116 ceph-mon[29897]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi116.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:58:57.414 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:57 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:57.414 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:57 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:58:57.414 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:57 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/2402441666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:57.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:57 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:57.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:57 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:57.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:57 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:58:57.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:57 smithi012 ceph-mon[26617]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:58:57.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:57 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi116.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:58:57.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:57 smithi012 ceph-mon[26617]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi116.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:58:57.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:57 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:57.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:57 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:58:57.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:57 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/2402441666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:57.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:57 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:57.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:57 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:57.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:57 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:58:57.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:57 smithi012 ceph-mon[37567]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:58:57.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:57 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi116.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:58:57.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:57 smithi012 ceph-mon[37567]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi116.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:58:57.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:57 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:58:57.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:57 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:58:57.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:57 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/2402441666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:58.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:58 smithi116 ceph-mon[29897]: pgmap v12: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:58:58.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:58 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:58:58.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:58 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:58:58.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:58 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:58:58.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:58 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:58:58.512 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:58 smithi012 ceph-mon[26617]: pgmap v12: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:58:58.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:58 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:58:58.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:58 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:58:58.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:58 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:58:58.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:58 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:58:58.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:58 smithi012 ceph-mon[37567]: pgmap v12: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:58:58.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:58 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:58:58.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:58 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:58:58.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:58 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:58:58.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:58 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:58:59.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:59 smithi116 ceph-mon[29897]: Upgrade: Updating mgr.x 2024-04-26T04:58:59.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:59 smithi116 ceph-mon[29897]: Deploying daemon mgr.x on smithi116 2024-04-26T04:58:59.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:58:59 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/559486464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:59.242 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:58 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 3. 2024-04-26T04:58:59.242 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:58 smithi116 systemd[1]: Stopped Ceph grafana.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:58:59.242 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:58 smithi116 systemd[1]: Starting Ceph grafana.a for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:58:59.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:59 smithi012 ceph-mon[26617]: Upgrade: Updating mgr.x 2024-04-26T04:58:59.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:59 smithi012 ceph-mon[26617]: Deploying daemon mgr.x on smithi116 2024-04-26T04:58:59.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:58:59 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/559486464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:59.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:59 smithi012 ceph-mon[37567]: Upgrade: Updating mgr.x 2024-04-26T04:58:59.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:59 smithi012 ceph-mon[37567]: Deploying daemon mgr.x on smithi116 2024-04-26T04:58:59.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:58:59 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/559486464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:58:59.975 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:58:59 smithi116 systemd[1]: Stopping Ceph mgr.x for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:58:59.976 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:59 smithi116 podman[75601]: 2024-04-26 04:58:59.576735851 +0000 UTC m=+0.024357949 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:58:59.976 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:59 smithi116 podman[75601]: 2024-04-26 04:58:59.722826728 +0000 UTC m=+0.170448806 container create a21281941280967bc49521e7236bc43180c3e6d1a0715d9a31ca85c2921da27a (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.openshift.expose-services=, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, distribution-scope=public, io.buildah.version=1.32.0, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, vcs-type=git) 2024-04-26T04:59:00.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:00 smithi116 ceph-mon[29897]: pgmap v13: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:00.242 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:59 smithi116 podman[75601]: 2024-04-26 04:58:59.975179123 +0000 UTC m=+0.422801213 container init a21281941280967bc49521e7236bc43180c3e6d1a0715d9a31ca85c2921da27a (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, architecture=x86_64, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, distribution-scope=public, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, vcs-type=git, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc.) 2024-04-26T04:59:00.242 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:58:59 smithi116 podman[75601]: 2024-04-26 04:58:59.978207104 +0000 UTC m=+0.425829191 container start a21281941280967bc49521e7236bc43180c3e6d1a0715d9a31ca85c2921da27a (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, build-date=2022-04-29T11:48:11.055897, architecture=x86_64, io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, release=239.1651231664, vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.buildah.version=1.32.0, description=Ceph Grafana Container, distribution-scope=public, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, vendor=Red Hat, Inc.) 2024-04-26T04:59:00.242 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:00 smithi116 bash[75601]: a21281941280967bc49521e7236bc43180c3e6d1a0715d9a31ca85c2921da27a 2024-04-26T04:59:00.242 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:00 smithi116 systemd[1]: Started Ceph grafana.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:59:00.242 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:00 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[75631]: logger=settings t=2024-04-26T04:59:00.130487718Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:59:00.243 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:00 smithi116 podman[75661]: 2024-04-26 04:59:00.160961486 +0000 UTC m=+0.022195095 container died a21281941280967bc49521e7236bc43180c3e6d1a0715d9a31ca85c2921da27a (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, io.buildah.version=1.32.0, maintainer=Paul Cuzner , description=Ceph Grafana Container, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, name=ubi8, com.redhat.component=ubi8-container, version=8.5, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8) 2024-04-26T04:59:00.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:00 smithi012 ceph-mon[26617]: pgmap v13: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:00.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:00 smithi012 ceph-mon[37567]: pgmap v13: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:01.174 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:00 smithi116 podman[75661]: 2024-04-26 04:59:00.837861068 +0000 UTC m=+0.699094652 container remove a21281941280967bc49521e7236bc43180c3e6d1a0715d9a31ca85c2921da27a (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, vcs-type=git, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, release=239.1651231664, version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, maintainer=Paul Cuzner , name=ubi8, io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, distribution-scope=public) 2024-04-26T04:59:01.175 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:00 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:59:01.175 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:00 smithi116 podman[75662]: 2024-04-26 04:59:00.906289201 +0000 UTC m=+0.766660976 container died b8c790c0f9e49defff228c4aba835a009bf72383c141a7b8a4cec76c3dfd3d40 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T04:59:01.491 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:01 smithi116 podman[75662]: 2024-04-26 04:59:01.259161998 +0000 UTC m=+1.119533781 container cleanup b8c790c0f9e49defff228c4aba835a009bf72383c141a7b8a4cec76c3dfd3d40 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb) 2024-04-26T04:59:01.491 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:01 smithi116 bash[75662]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x 2024-04-26T04:59:01.823 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:01 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:59:01.823 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:01 smithi116 podman[75692]: 2024-04-26 04:59:01.516522235 +0000 UTC m=+1.040119734 container remove b8c790c0f9e49defff228c4aba835a009bf72383c141a7b8a4cec76c3dfd3d40 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T04:59:01.823 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:01 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-26T04:59:01.824 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:01 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-04-26T04:59:01.824 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:01 smithi116 systemd[1]: Stopped Ceph mgr.x for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:59:01.824 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:01 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mgr.x.service: Consumed 26.295s CPU time. 2024-04-26T04:59:02.145 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:01 smithi116 systemd[1]: Starting Ceph mgr.x for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:59:02.145 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:02 smithi116 podman[75855]: 2024-04-26 04:59:01.940732438 +0000 UTC m=+0.018017654 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:59:02.145 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:02 smithi116 podman[75855]: 2024-04-26 04:59:02.042502778 +0000 UTC m=+0.119787996 container create 5499bf989999b867b9e7b18933d59ab1e040e2fa379f78acdac5bc634799c3dc (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x, RELEASE=squid-b22e2eb, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:59:02.451 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:02 smithi116 ceph-mon[29897]: pgmap v14: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:02.452 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:02 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/3499278132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:02.452 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:02 smithi116 podman[75855]: 2024-04-26 04:59:02.298802397 +0000 UTC m=+0.376087612 container init 5499bf989999b867b9e7b18933d59ab1e040e2fa379f78acdac5bc634799c3dc (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-04-26T04:59:02.452 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:02 smithi116 podman[75855]: 2024-04-26 04:59:02.301803034 +0000 UTC m=+0.379088251 container start 5499bf989999b867b9e7b18933d59ab1e040e2fa379f78acdac5bc634799c3dc (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T04:59:02.452 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:02 smithi116 ceph-mgr[75874]: -- 172.21.15.116:0/641350663 <== mon.1 v2:172.21.15.116:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x561bc53505a0 con 0x561bc5325000 2024-04-26T04:59:02.452 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:02 smithi116 bash[75855]: 5499bf989999b867b9e7b18933d59ab1e040e2fa379f78acdac5bc634799c3dc 2024-04-26T04:59:02.452 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:02 smithi116 systemd[1]: Started Ceph mgr.x for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:59:02.452 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:02 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:02.450+0000 7f9ed5ee1200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:59:02.512 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:02 smithi012 ceph-mon[26617]: pgmap v14: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:02.512 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:02 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/3499278132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:02.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:02 smithi012 ceph-mon[37567]: pgmap v14: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:02.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:02 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/3499278132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:02.741 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:02 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:02.641+0000 7f9ed5ee1200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:59:03.092 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:02 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:02.746+0000 7f9ed5ee1200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:59:03.093 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:03 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:03.091+0000 7f9ed5ee1200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:59:03.360 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:03 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:03.189+0000 7f9ed5ee1200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:59:03.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:03 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:59:03.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:03 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:59:03.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:03 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:03.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:03 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:59:03.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:03 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:59:03.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:03 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:03.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:03 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:59:03.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:03 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:59:03.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:03 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:04.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:04 smithi116 ceph-mon[29897]: pgmap v15: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:04.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:04 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/2415125143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:04.742 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:04 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:04.434+0000 7f9ed5ee1200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:59:04.742 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:04 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:04.618+0000 7f9ed5ee1200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:59:04.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:04 smithi012 ceph-mon[26617]: pgmap v15: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:04.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:04 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/2415125143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:04.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:04 smithi012 ceph-mon[37567]: pgmap v15: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:04.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:04 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/2415125143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:05.071 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:04 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:04.818+0000 7f9ed5ee1200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:59:05.420 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:05 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:05.121+0000 7f9ed5ee1200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:59:05.420 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:05 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:05.215+0000 7f9ed5ee1200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:59:05.420 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:05 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:05.419+0000 7f9ed5ee1200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:59:05.741 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:05 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:05.638+0000 7f9ed5ee1200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:59:06.730 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:06 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:06.278+0000 7f9ed5ee1200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:59:06.731 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:06 smithi116 ceph-mon[29897]: pgmap v16: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:06.731 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:06 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/412861754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:06.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:06 smithi012 ceph-mon[26617]: pgmap v16: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:06.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:06 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/412861754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:06.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:06 smithi012 ceph-mon[37567]: pgmap v16: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:06.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:06 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/412861754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:06.991 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:06 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:06.729+0000 7f9ed5ee1200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:59:06.991 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:06 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:06.830+0000 7f9ed5ee1200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:59:07.491 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:07 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:07.044+0000 7f9ed5ee1200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:59:07.955 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:07 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:59:07.955 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:07 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:59:07.956 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:07 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:07.598+0000 7f9ed5ee1200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:59:07.956 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:07 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:07.695+0000 7f9ed5ee1200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:59:07.991 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:07 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:59:07.992 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:07 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:59:07.992 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:07 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:59:07.992 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:07 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:59:08.216 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:07 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:07.954+0000 7f9ed5ee1200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:59:08.216 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:08 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:08.097+0000 7f9ed5ee1200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:59:08.216 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:08 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:08.214+0000 7f9ed5ee1200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:59:08.262 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:07 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:59:07] "GET /metrics HTTP/1.1" 200 33778 "" "Prometheus/2.43.0" 2024-04-26T04:59:08.467 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:08 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:08.332+0000 7f9ed5ee1200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:59:08.467 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:08 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:08.427+0000 7f9ed5ee1200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:59:08.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:08 smithi116 ceph-mon[29897]: pgmap v17: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:08.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:08 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:59:08.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:08 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:59:08.992 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:59:08 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T04:59:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi116.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:59:09.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:08 smithi012 ceph-mon[26617]: pgmap v17: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:09.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:08 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:59:09.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:08 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:59:09.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:08 smithi012 ceph-mon[37567]: pgmap v17: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:09.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:08 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:59:09.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:08 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:59:09.526 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:09 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:09.248+0000 7f9ed5ee1200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:59:09.526 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:09 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:09.482+0000 7f9ed5ee1200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:59:09.526 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:09 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: [26/Apr/2024:04:59:09] ENGINE Bus STARTING 2024-04-26T04:59:09.792 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:09 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/3952712816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:09.792 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:09 smithi116 ceph-mon[29897]: Standby manager daemon x restarted 2024-04-26T04:59:09.792 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:09 smithi116 ceph-mon[29897]: Standby manager daemon x started 2024-04-26T04:59:09.792 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:09 smithi116 ceph-mon[29897]: from='mgr.? 172.21.15.116:0/3292616391' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:59:09.792 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:09 smithi116 ceph-mon[29897]: from='mgr.? 172.21.15.116:0/3292616391' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:59:09.792 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:09 smithi116 ceph-mon[29897]: from='mgr.? 172.21.15.116:0/3292616391' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:59:09.792 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:09 smithi116 ceph-mon[29897]: from='mgr.? 172.21.15.116:0/3292616391' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:59:09.793 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:09 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: CherryPy Checker: 2024-04-26T04:59:09.793 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:09 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: The Application mounted at '' has an empty config. 2024-04-26T04:59:09.793 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:09 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:09.793 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:09 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: [26/Apr/2024:04:59:09] ENGINE Serving on http://:::9283 2024-04-26T04:59:09.793 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:09 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: [26/Apr/2024:04:59:09] ENGINE Bus STARTED 2024-04-26T04:59:10.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:09 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/3952712816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:10.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:09 smithi012 ceph-mon[26617]: Standby manager daemon x restarted 2024-04-26T04:59:10.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:09 smithi012 ceph-mon[26617]: Standby manager daemon x started 2024-04-26T04:59:10.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:09 smithi012 ceph-mon[26617]: from='mgr.? 172.21.15.116:0/3292616391' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:59:10.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:09 smithi012 ceph-mon[26617]: from='mgr.? 172.21.15.116:0/3292616391' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:59:10.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:09 smithi012 ceph-mon[26617]: from='mgr.? 172.21.15.116:0/3292616391' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:59:10.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:09 smithi012 ceph-mon[26617]: from='mgr.? 172.21.15.116:0/3292616391' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:59:10.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:09 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/3952712816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:10.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:09 smithi012 ceph-mon[37567]: Standby manager daemon x restarted 2024-04-26T04:59:10.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:09 smithi012 ceph-mon[37567]: Standby manager daemon x started 2024-04-26T04:59:10.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:09 smithi012 ceph-mon[37567]: from='mgr.? 172.21.15.116:0/3292616391' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:59:10.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:09 smithi012 ceph-mon[37567]: from='mgr.? 172.21.15.116:0/3292616391' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:59:10.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:09 smithi012 ceph-mon[37567]: from='mgr.? 172.21.15.116:0/3292616391' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:59:10.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:09 smithi012 ceph-mon[37567]: from='mgr.? 172.21.15.116:0/3292616391' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:59:10.918 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:10 smithi116 ceph-mon[29897]: pgmap v18: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:10.918 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:10 smithi116 ceph-mon[29897]: mgrmap e32: y(active, since 29s), standbys: x 2024-04-26T04:59:11.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:10 smithi012 ceph-mon[26617]: pgmap v18: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:11.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:10 smithi012 ceph-mon[26617]: mgrmap e32: y(active, since 29s), standbys: x 2024-04-26T04:59:11.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:10 smithi012 ceph-mon[37567]: pgmap v18: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:11.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:10 smithi012 ceph-mon[37567]: mgrmap e32: y(active, since 29s), standbys: x 2024-04-26T04:59:11.970 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:11 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 4. 2024-04-26T04:59:11.970 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:11 smithi116 systemd[1]: Stopped Ceph grafana.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:59:11.970 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:11 smithi116 systemd[1]: Starting Ceph grafana.a for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:59:11.970 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:11 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:11.971 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:11 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/992154565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:11.971 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:11 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:59:11.971 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:11 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:59:11.971 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:11 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:59:11.971 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:11 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:59:11.971 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:11 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:59:11.971 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:11 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:11.971 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:11 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:59:11.971 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:11 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:59:11.971 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:11 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-26T04:59:11.971 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:11 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-26T04:59:11.971 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:11 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-26T04:59:11.972 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:11 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-26T04:59:11.972 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:11 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-26T04:59:11.972 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:11 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-26T04:59:11.972 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:11 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T04:59:11.972 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:11 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-26T04:59:12.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:11 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:12.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:11 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/992154565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:12.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:11 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:59:12.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:11 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:59:12.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:11 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:59:12.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:11 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:59:12.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:11 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:59:12.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:11 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:12.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:11 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:59:12.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:11 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:59:12.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:11 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-26T04:59:12.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:11 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-26T04:59:12.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:11 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-26T04:59:12.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:11 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-26T04:59:12.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:11 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-26T04:59:12.015 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:11 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-26T04:59:12.015 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:11 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T04:59:12.015 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:11 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-26T04:59:12.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:11 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:12.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:11 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/992154565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:12.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:11 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:59:12.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:11 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:59:12.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:11 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:59:12.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:11 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:59:12.016 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:11 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:59:12.016 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:11 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:12.016 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:11 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:59:12.016 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:11 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:59:12.016 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:11 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-26T04:59:12.016 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:11 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-26T04:59:12.016 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:11 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-26T04:59:12.016 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:11 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-26T04:59:12.016 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:11 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-26T04:59:12.016 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:11 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-26T04:59:12.016 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:11 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T04:59:12.016 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:11 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-26T04:59:12.241 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:11 smithi116 podman[79325]: 2024-04-26 04:59:11.871386086 +0000 UTC m=+0.017114050 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:59:12.241 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:11 smithi116 podman[79325]: 2024-04-26 04:59:11.999606681 +0000 UTC m=+0.145334635 container create 48405cc7cc8c306a29fa96fc12785f32e89bf4ad8e9e8136a132f5359d728a66 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, vendor=Red Hat, Inc., io.openshift.expose-services=, name=ubi8, architecture=x86_64, release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, version=8.5, description=Ceph Grafana Container, io.buildah.version=1.32.0, vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container) 2024-04-26T04:59:12.242 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:12 smithi116 podman[79325]: 2024-04-26 04:59:12.196899034 +0000 UTC m=+0.342626989 container init 48405cc7cc8c306a29fa96fc12785f32e89bf4ad8e9e8136a132f5359d728a66 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, io.openshift.tags=base rhel8, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.buildah.version=1.32.0, vcs-type=git, build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, vendor=Red Hat, Inc., maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-04-26T04:59:12.242 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:12 smithi116 podman[79325]: 2024-04-26 04:59:12.19934505 +0000 UTC m=+0.345073013 container start 48405cc7cc8c306a29fa96fc12785f32e89bf4ad8e9e8136a132f5359d728a66 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., maintainer=Paul Cuzner , vcs-type=git, distribution-scope=public, io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, io.openshift.tags=base rhel8, release=239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, name=ubi8, description=Ceph Grafana Container) 2024-04-26T04:59:12.581 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:12 smithi116 bash[79325]: 48405cc7cc8c306a29fa96fc12785f32e89bf4ad8e9e8136a132f5359d728a66 2024-04-26T04:59:12.581 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:12 smithi116 systemd[1]: Started Ceph grafana.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:59:12.581 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:12 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[79340]: logger=settings t=2024-04-26T04:59:12.34722838Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:59:12.581 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:12 smithi116 podman[79357]: 2024-04-26 04:59:12.378284175 +0000 UTC m=+0.018891773 container died 48405cc7cc8c306a29fa96fc12785f32e89bf4ad8e9e8136a132f5359d728a66 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, name=ubi8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, release=239.1651231664, maintainer=Paul Cuzner , vendor=Red Hat, Inc., description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, io.openshift.expose-services=, vcs-type=git, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5) 2024-04-26T04:59:12.933 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:12 smithi116 ceph-mon[29897]: pgmap v19: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:12.934 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:12 smithi116 ceph-mon[29897]: pgmap v20: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:12.934 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:12 smithi116 ceph-mon[29897]: Upgrade: Setting container_image for all mgr 2024-04-26T04:59:12.934 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:12 smithi116 ceph-mon[29897]: Upgrade: It appears safe to stop mon.a 2024-04-26T04:59:13.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:12 smithi012 ceph-mon[37567]: pgmap v19: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:13.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:12 smithi012 ceph-mon[37567]: pgmap v20: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:13.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:12 smithi012 ceph-mon[37567]: Upgrade: Setting container_image for all mgr 2024-04-26T04:59:13.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:12 smithi012 ceph-mon[37567]: Upgrade: It appears safe to stop mon.a 2024-04-26T04:59:13.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:12 smithi012 ceph-mon[26617]: pgmap v19: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:13.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:12 smithi012 ceph-mon[26617]: pgmap v20: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:13.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:12 smithi012 ceph-mon[26617]: Upgrade: Setting container_image for all mgr 2024-04-26T04:59:13.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:12 smithi012 ceph-mon[26617]: Upgrade: It appears safe to stop mon.a 2024-04-26T04:59:13.241 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:12 smithi116 podman[79357]: 2024-04-26 04:59:12.93350205 +0000 UTC m=+0.574109664 container remove 48405cc7cc8c306a29fa96fc12785f32e89bf4ad8e9e8136a132f5359d728a66 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, io.openshift.tags=base rhel8, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public, com.redhat.component=ubi8-container, vcs-type=git, io.buildah.version=1.32.0, name=ubi8, io.openshift.expose-services=, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-04-26T04:59:13.242 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:12 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:59:13.242 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:13 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:59:14.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:14 smithi116 ceph-mon[29897]: Upgrade: Updating mon.a 2024-04-26T04:59:14.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:14 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:59:14.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:14 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:59:14.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:14 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:59:14.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:14 smithi116 ceph-mon[29897]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:59:14.743 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:14 smithi116 ceph-mon[29897]: Deploying daemon mon.a on smithi012 2024-04-26T04:59:14.743 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:14 smithi116 ceph-mon[29897]: pgmap v21: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:14.743 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:14 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/2561389253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:14.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:14 smithi012 ceph-mon[26617]: Upgrade: Updating mon.a 2024-04-26T04:59:14.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:14 smithi012 ceph-mon[26617]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:59:14.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:14 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:59:14.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:14 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:59:14.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:14 smithi012 ceph-mon[26617]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:59:14.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:14 smithi012 ceph-mon[26617]: Deploying daemon mon.a on smithi012 2024-04-26T04:59:14.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:14 smithi012 ceph-mon[26617]: pgmap v21: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:14.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:14 smithi012 ceph-mon[26617]: from='client.? 172.21.15.12:0/2561389253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:14.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:14 smithi012 ceph-mon[37567]: Upgrade: Updating mon.a 2024-04-26T04:59:14.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:14 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='mgr.y' 2024-04-26T04:59:14.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:14 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:59:14.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:14 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:59:14.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:14 smithi012 ceph-mon[37567]: from='mgr.15180 172.21.15.12:0/4037275184' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:59:14.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:14 smithi012 ceph-mon[37567]: Deploying daemon mon.a on smithi012 2024-04-26T04:59:14.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:14 smithi012 ceph-mon[37567]: pgmap v21: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:14.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:14 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/2561389253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:15.483 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:15 smithi012 systemd[1]: Stopping Ceph mon.a for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:59:15.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:15 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-a[26613]: 2024-04-26T04:59:15.481+0000 7fc2c6a20700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:59:15.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:15 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-a[26613]: 2024-04-26T04:59:15.481+0000 7fc2c6a20700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-04-26T04:59:15.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:15 smithi012 podman[91664]: 2024-04-26 04:59:15.717119956 +0000 UTC m=+0.346197461 container died 1b1568ad49e73ce545a5c9e562b88e4f3f39191857e81d1d96b94520b6769f62 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-a, RELEASE=quincy-c61472c, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-04-26T04:59:16.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:16 smithi012 podman[91664]: 2024-04-26 04:59:16.149216562 +0000 UTC m=+0.778294066 container cleanup 1b1568ad49e73ce545a5c9e562b88e4f3f39191857e81d1d96b94520b6769f62 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-a, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=quincy-c61472c, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-04-26T04:59:16.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:16 smithi012 bash[91664]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-a 2024-04-26T04:59:16.734 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:16 smithi012 podman[91677]: 2024-04-26 04:59:16.309029668 +0000 UTC m=+0.586959029 container remove 1b1568ad49e73ce545a5c9e562b88e4f3f39191857e81d1d96b94520b6769f62 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-a, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, ceph=True, maintainer=Guillaume Abrioux , RELEASE=quincy-c61472c, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-04-26T04:59:16.734 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:16 smithi012 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mon.a.service: Deactivated successfully. 2024-04-26T04:59:16.734 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:16 smithi012 systemd[1]: Stopped Ceph mon.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:59:16.734 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:16 smithi012 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mon.a.service: Consumed 13.920s CPU time. 2024-04-26T04:59:17.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:16 smithi012 systemd[1]: Starting Ceph mon.a for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:59:17.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:16 smithi012 podman[91833]: 2024-04-26 04:59:16.855792286 +0000 UTC m=+0.018496049 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:59:17.357 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 podman[91833]: 2024-04-26 04:59:17.101852376 +0000 UTC m=+0.264556112 container create db1f873383b097011eb4f61089abb64565ce058ed9342ec7badb8ba4c09eb414 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-a, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, io.buildah.version=1.33.5) 2024-04-26T04:59:17.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 podman[91833]: 2024-04-26 04:59:17.357167767 +0000 UTC m=+0.519871505 container init db1f873383b097011eb4f61089abb64565ce058ed9342ec7badb8ba4c09eb414 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-a, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-04-26T04:59:17.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 podman[91833]: 2024-04-26 04:59:17.360491328 +0000 UTC m=+0.523195062 container start db1f873383b097011eb4f61089abb64565ce058ed9342ec7badb8ba4c09eb414 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-a, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424) 2024-04-26T04:59:17.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: set uid:gid to 167:167 (ceph:ceph) 2024-04-26T04:59:17.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev), process ceph-mon, pid 2 2024-04-26T04:59:17.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: pidfile_write: ignore empty --pid-file 2024-04-26T04:59:17.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: load: jerasure load: lrc 2024-04-26T04:59:17.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: RocksDB version: 7.9.2 2024-04-26T04:59:17.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Git sha 0 2024-04-26T04:59:17.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Compile date 2024-04-25 16:43:49 2024-04-26T04:59:17.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: DB SUMMARY 2024-04-26T04:59:17.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: DB Session ID: GCMO3EZTEIXDLVET00O7 2024-04-26T04:59:17.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: CURRENT file: CURRENT 2024-04-26T04:59:17.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: IDENTITY file: IDENTITY 2024-04-26T04:59:17.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: MANIFEST file: MANIFEST-000015 size: 972 Bytes 2024-04-26T04:59:17.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000030.sst 2024-04-26T04:59:17.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000028.log size: 2550582 ; 2024-04-26T04:59:17.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.error_if_exists: 0 2024-04-26T04:59:17.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.create_if_missing: 0 2024-04-26T04:59:17.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.paranoid_checks: 1 2024-04-26T04:59:17.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-26T04:59:17.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-26T04:59:17.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-26T04:59:17.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.env: 0x5574d36f7c60 2024-04-26T04:59:17.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.fs: PosixFileSystem 2024-04-26T04:59:17.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.info_log: 0x5574d55d3aa0 2024-04-26T04:59:17.765 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.max_file_opening_threads: 16 2024-04-26T04:59:17.765 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.statistics: (nil) 2024-04-26T04:59:17.766 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.use_fsync: 0 2024-04-26T04:59:17.766 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.max_log_file_size: 0 2024-04-26T04:59:17.766 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-26T04:59:17.766 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-26T04:59:17.766 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.keep_log_file_num: 1000 2024-04-26T04:59:17.766 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.recycle_log_file_num: 0 2024-04-26T04:59:17.766 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.allow_fallocate: 1 2024-04-26T04:59:17.767 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.allow_mmap_reads: 0 2024-04-26T04:59:17.767 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.allow_mmap_writes: 0 2024-04-26T04:59:17.767 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.use_direct_reads: 0 2024-04-26T04:59:17.767 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T04:59:17.767 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T04:59:17.767 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.db_log_dir: 2024-04-26T04:59:17.767 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.wal_dir: 2024-04-26T04:59:17.767 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T04:59:17.767 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T04:59:17.767 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T04:59:17.767 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T04:59:17.767 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T04:59:17.768 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T04:59:17.768 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T04:59:17.768 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T04:59:17.768 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.write_buffer_manager: 0x5574d55a6820 2024-04-26T04:59:17.768 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T04:59:17.768 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T04:59:17.768 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T04:59:17.768 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T04:59:17.768 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T04:59:17.768 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T04:59:17.768 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T04:59:17.768 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T04:59:17.768 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.unordered_write: 0 2024-04-26T04:59:17.768 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T04:59:17.769 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T04:59:17.769 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T04:59:17.769 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T04:59:17.769 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.row_cache: None 2024-04-26T04:59:17.769 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.wal_filter: None 2024-04-26T04:59:17.769 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T04:59:17.769 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T04:59:17.769 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.two_write_queues: 0 2024-04-26T04:59:17.769 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T04:59:17.769 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.wal_compression: 0 2024-04-26T04:59:17.769 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.atomic_flush: 0 2024-04-26T04:59:17.769 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T04:59:17.769 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T04:59:17.769 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T04:59:17.769 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.log_readahead_size: 0 2024-04-26T04:59:17.770 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T04:59:17.770 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T04:59:17.770 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T04:59:17.770 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T04:59:17.770 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T04:59:17.770 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T04:59:17.770 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-26T04:59:17.770 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.max_background_jobs: 2 2024-04-26T04:59:17.770 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.max_background_compactions: -1 2024-04-26T04:59:17.770 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.max_subcompactions: 1 2024-04-26T04:59:17.770 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T04:59:17.770 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T04:59:17.770 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T04:59:17.770 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T04:59:17.770 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T04:59:17.771 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T04:59:17.771 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T04:59:17.771 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T04:59:17.771 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.max_open_files: -1 2024-04-26T04:59:17.771 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T04:59:17.771 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T04:59:17.771 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T04:59:17.771 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T04:59:17.771 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.max_background_flushes: -1 2024-04-26T04:59:17.771 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Compression algorithms supported: 2024-04-26T04:59:17.771 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T04:59:17.771 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: kZSTD supported: 0 2024-04-26T04:59:17.771 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: kXpressCompression supported: 0 2024-04-26T04:59:17.771 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T04:59:17.771 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: kZlibCompression supported: 1 2024-04-26T04:59:17.772 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: kSnappyCompression supported: 1 2024-04-26T04:59:17.772 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: kLZ4Compression supported: 1 2024-04-26T04:59:17.772 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: kBZip2Compression supported: 0 2024-04-26T04:59:17.772 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T04:59:17.772 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-26T04:59:17.772 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-04-26T04:59:17.772 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-26T04:59:17.772 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T04:59:17.772 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.merge_operator: 2024-04-26T04:59:17.772 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.compaction_filter: None 2024-04-26T04:59:17.772 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T04:59:17.772 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T04:59:17.772 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T04:59:17.772 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T04:59:17.773 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5574d55d3ba0) 2024-04-26T04:59:17.773 INFO:journalctl@ceph.mon.a.smithi012.stdout: cache_index_and_filter_blocks: 1 2024-04-26T04:59:17.773 INFO:journalctl@ceph.mon.a.smithi012.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T04:59:17.773 INFO:journalctl@ceph.mon.a.smithi012.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T04:59:17.773 INFO:journalctl@ceph.mon.a.smithi012.stdout: pin_top_level_index_and_filter: 1 2024-04-26T04:59:17.773 INFO:journalctl@ceph.mon.a.smithi012.stdout: index_type: 0 2024-04-26T04:59:17.773 INFO:journalctl@ceph.mon.a.smithi012.stdout: data_block_index_type: 0 2024-04-26T04:59:17.773 INFO:journalctl@ceph.mon.a.smithi012.stdout: index_shortening: 1 2024-04-26T04:59:17.773 INFO:journalctl@ceph.mon.a.smithi012.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T04:59:17.773 INFO:journalctl@ceph.mon.a.smithi012.stdout: checksum: 4 2024-04-26T04:59:17.773 INFO:journalctl@ceph.mon.a.smithi012.stdout: no_block_cache: 0 2024-04-26T04:59:17.773 INFO:journalctl@ceph.mon.a.smithi012.stdout: block_cache: 0x5574d55d7090 2024-04-26T04:59:17.773 INFO:journalctl@ceph.mon.a.smithi012.stdout: block_cache_name: BinnedLRUCache 2024-04-26T04:59:17.774 INFO:journalctl@ceph.mon.a.smithi012.stdout: block_cache_options: 2024-04-26T04:59:17.774 INFO:journalctl@ceph.mon.a.smithi012.stdout: capacity : 536870912 2024-04-26T04:59:17.774 INFO:journalctl@ceph.mon.a.smithi012.stdout: num_shard_bits : 4 2024-04-26T04:59:17.774 INFO:journalctl@ceph.mon.a.smithi012.stdout: strict_capacity_limit : 0 2024-04-26T04:59:17.774 INFO:journalctl@ceph.mon.a.smithi012.stdout: high_pri_pool_ratio: 0.000 2024-04-26T04:59:17.774 INFO:journalctl@ceph.mon.a.smithi012.stdout: block_cache_compressed: (nil) 2024-04-26T04:59:17.774 INFO:journalctl@ceph.mon.a.smithi012.stdout: persistent_cache: (nil) 2024-04-26T04:59:17.774 INFO:journalctl@ceph.mon.a.smithi012.stdout: block_size: 4096 2024-04-26T04:59:17.774 INFO:journalctl@ceph.mon.a.smithi012.stdout: block_size_deviation: 10 2024-04-26T04:59:17.774 INFO:journalctl@ceph.mon.a.smithi012.stdout: block_restart_interval: 16 2024-04-26T04:59:17.774 INFO:journalctl@ceph.mon.a.smithi012.stdout: index_block_restart_interval: 1 2024-04-26T04:59:17.774 INFO:journalctl@ceph.mon.a.smithi012.stdout: metadata_block_size: 4096 2024-04-26T04:59:17.774 INFO:journalctl@ceph.mon.a.smithi012.stdout: partition_filters: 0 2024-04-26T04:59:17.775 INFO:journalctl@ceph.mon.a.smithi012.stdout: use_delta_encoding: 1 2024-04-26T04:59:17.775 INFO:journalctl@ceph.mon.a.smithi012.stdout: filter_policy: bloomfilter 2024-04-26T04:59:17.775 INFO:journalctl@ceph.mon.a.smithi012.stdout: whole_key_filtering: 1 2024-04-26T04:59:17.775 INFO:journalctl@ceph.mon.a.smithi012.stdout: verify_compression: 0 2024-04-26T04:59:17.775 INFO:journalctl@ceph.mon.a.smithi012.stdout: read_amp_bytes_per_bit: 0 2024-04-26T04:59:17.775 INFO:journalctl@ceph.mon.a.smithi012.stdout: format_version: 5 2024-04-26T04:59:17.775 INFO:journalctl@ceph.mon.a.smithi012.stdout: enable_index_compression: 1 2024-04-26T04:59:17.775 INFO:journalctl@ceph.mon.a.smithi012.stdout: block_align: 0 2024-04-26T04:59:17.775 INFO:journalctl@ceph.mon.a.smithi012.stdout: max_auto_readahead_size: 262144 2024-04-26T04:59:17.775 INFO:journalctl@ceph.mon.a.smithi012.stdout: prepopulate_block_cache: 0 2024-04-26T04:59:17.775 INFO:journalctl@ceph.mon.a.smithi012.stdout: initial_auto_readahead_size: 8192 2024-04-26T04:59:17.775 INFO:journalctl@ceph.mon.a.smithi012.stdout: num_file_reads_for_auto_readahead: 2 2024-04-26T04:59:17.775 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T04:59:17.776 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T04:59:17.776 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.compression: NoCompression 2024-04-26T04:59:17.776 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T04:59:17.776 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T04:59:17.776 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T04:59:17.776 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.num_levels: 7 2024-04-26T04:59:17.776 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T04:59:17.776 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T04:59:17.776 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T04:59:17.776 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T04:59:17.776 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T04:59:17.776 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T04:59:17.776 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T04:59:17.777 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:59:17.777 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T04:59:17.777 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T04:59:17.777 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-26T04:59:17.777 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-26T04:59:17.777 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T04:59:17.777 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T04:59:17.777 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T04:59:17.777 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T04:59:17.777 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:59:17.777 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-26T04:59:17.777 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T04:59:17.777 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T04:59:17.778 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-26T04:59:17.778 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T04:59:17.778 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T04:59:17.778 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T04:59:17.778 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T04:59:17.778 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T04:59:17.778 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T04:59:17.778 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T04:59:17.778 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T04:59:17.778 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T04:59:17.778 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T04:59:17.778 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T04:59:17.778 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T04:59:17.778 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T04:59:17.779 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T04:59:17.779 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T04:59:17.779 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T04:59:17.779 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T04:59:17.779 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-26T04:59:17.779 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.arena_block_size: 1048576 2024-04-26T04:59:17.779 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T04:59:17.779 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T04:59:17.779 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T04:59:17.779 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T04:59:17.779 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T04:59:17.779 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T04:59:17.779 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T04:59:17.779 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T04:59:17.779 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T04:59:17.779 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T04:59:17.780 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T04:59:17.780 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T04:59:17.780 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T04:59:17.780 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.table_properties_collectors: 2024-04-26T04:59:17.780 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.inplace_update_support: 0 2024-04-26T04:59:17.780 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T04:59:17.780 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T04:59:17.780 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T04:59:17.780 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T04:59:17.780 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.bloom_locality: 0 2024-04-26T04:59:17.780 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.max_successive_merges: 0 2024-04-26T04:59:17.780 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T04:59:17.780 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T04:59:17.780 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T04:59:17.781 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T04:59:17.781 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.ttl: 2592000 2024-04-26T04:59:17.781 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T04:59:17.781 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-26T04:59:17.781 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-26T04:59:17.781 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.enable_blob_files: false 2024-04-26T04:59:17.781 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.min_blob_size: 0 2024-04-26T04:59:17.781 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T04:59:17.781 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T04:59:17.781 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T04:59:17.781 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T04:59:17.781 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-26T04:59:17.781 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-26T04:59:17.781 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.blob_file_starting_level: 0 2024-04-26T04:59:17.781 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-26T04:59:17.782 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-04-26T04:59:17.782 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T04:59:17.782 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 32, last_sequence is 10517, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T04:59:17.782 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-04-26T04:59:17.782 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 1d68161f-8608-4b5c-829e-6155786e8c51 2024-04-26T04:59:17.782 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714107557391780, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-04-26T04:59:17.782 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-04-26T04:59:17.782 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714107557399434, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 2333220, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 10518, "largest_seqno": 11692, "table_properties": {"data_size": 2327168, "index_size": 3625, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1413, "raw_key_size": 13630, "raw_average_key_size": 24, "raw_value_size": 2314924, "raw_average_value_size": 4141, "num_data_blocks": 167, "num_entries": 559, "num_filter_entries": 559, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1714107557, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "1d68161f-8608-4b5c-829e-6155786e8c51", "db_session_id": "GCMO3EZTEIXDLVET00O7", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-04-26T04:59:17.782 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714107557399526, "job": 1, "event": "recovery_finished"} 2024-04-26T04:59:17.782 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-04-26T04:59:17.782 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T04:59:17.782 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T04:59:17.782 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5574d56c8000 2024-04-26T04:59:17.783 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: DB pointer 0x5574d56b2000 2024-04-26T04:59:17.783 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-26T04:59:17.783 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-26T04:59:17.783 INFO:journalctl@ceph.mon.a.smithi012.stdout: ** DB Stats ** 2024-04-26T04:59:17.783 INFO:journalctl@ceph.mon.a.smithi012.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:59:17.783 INFO:journalctl@ceph.mon.a.smithi012.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T04:59:17.783 INFO:journalctl@ceph.mon.a.smithi012.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T04:59:17.783 INFO:journalctl@ceph.mon.a.smithi012.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:59:17.783 INFO:journalctl@ceph.mon.a.smithi012.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T04:59:17.783 INFO:journalctl@ceph.mon.a.smithi012.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T04:59:17.784 INFO:journalctl@ceph.mon.a.smithi012.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:59:17.784 INFO:journalctl@ceph.mon.a.smithi012.stdout: 2024-04-26T04:59:17.784 INFO:journalctl@ceph.mon.a.smithi012.stdout: ** Compaction Stats [default] ** 2024-04-26T04:59:17.784 INFO:journalctl@ceph.mon.a.smithi012.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T04:59:17.784 INFO:journalctl@ceph.mon.a.smithi012.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-26T04:59:17.784 INFO:journalctl@ceph.mon.a.smithi012.stdout: L0 1/0 2.23 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 593.8 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-26T04:59:17.784 INFO:journalctl@ceph.mon.a.smithi012.stdout: L6 1/0 6.24 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-26T04:59:17.784 INFO:journalctl@ceph.mon.a.smithi012.stdout: Sum 2/0 8.47 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 593.8 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-26T04:59:17.784 INFO:journalctl@ceph.mon.a.smithi012.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 593.8 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-26T04:59:17.784 INFO:journalctl@ceph.mon.a.smithi012.stdout: 2024-04-26T04:59:17.784 INFO:journalctl@ceph.mon.a.smithi012.stdout: ** Compaction Stats [default] ** 2024-04-26T04:59:17.784 INFO:journalctl@ceph.mon.a.smithi012.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T04:59:17.784 INFO:journalctl@ceph.mon.a.smithi012.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:59:17.785 INFO:journalctl@ceph.mon.a.smithi012.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 593.8 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-26T04:59:17.785 INFO:journalctl@ceph.mon.a.smithi012.stdout: 2024-04-26T04:59:17.785 INFO:journalctl@ceph.mon.a.smithi012.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-26T04:59:17.785 INFO:journalctl@ceph.mon.a.smithi012.stdout: 2024-04-26T04:59:17.785 INFO:journalctl@ceph.mon.a.smithi012.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:59:17.785 INFO:journalctl@ceph.mon.a.smithi012.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-04-26T04:59:17.785 INFO:journalctl@ceph.mon.a.smithi012.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T04:59:17.785 INFO:journalctl@ceph.mon.a.smithi012.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T04:59:17.785 INFO:journalctl@ceph.mon.a.smithi012.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T04:59:17.785 INFO:journalctl@ceph.mon.a.smithi012.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T04:59:17.785 INFO:journalctl@ceph.mon.a.smithi012.stdout: Cumulative compaction: 0.00 GB write, 184.69 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:59:17.785 INFO:journalctl@ceph.mon.a.smithi012.stdout: Interval compaction: 0.00 GB write, 184.69 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:59:17.785 INFO:journalctl@ceph.mon.a.smithi012.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T04:59:17.786 INFO:journalctl@ceph.mon.a.smithi012.stdout: Block cache BinnedLRUCache@0x5574d55d7090#2 capacity: 512.00 MB usage: 5.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-04-26T04:59:17.786 INFO:journalctl@ceph.mon.a.smithi012.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.53 KB,0.000292063%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2024-04-26T04:59:17.786 INFO:journalctl@ceph.mon.a.smithi012.stdout: 2024-04-26T04:59:17.786 INFO:journalctl@ceph.mon.a.smithi012.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T04:59:17.786 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: starting mon.a rank 0 at public addrs [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] at bind addrs [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T04:59:17.786 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: mon.a@-1(???) e3 preinit fsid 0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T04:59:17.786 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: mon.a@-1(???).mds e9 new map 2024-04-26T04:59:17.786 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: mon.a@-1(???).mds e9 print_map 2024-04-26T04:59:17.786 INFO:journalctl@ceph.mon.a.smithi012.stdout: e9 2024-04-26T04:59:17.786 INFO:journalctl@ceph.mon.a.smithi012.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T04:59:17.786 INFO:journalctl@ceph.mon.a.smithi012.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T04:59:17.786 INFO:journalctl@ceph.mon.a.smithi012.stdout: legacy client fscid: 1 2024-04-26T04:59:17.786 INFO:journalctl@ceph.mon.a.smithi012.stdout: 2024-04-26T04:59:17.787 INFO:journalctl@ceph.mon.a.smithi012.stdout: Filesystem 'foo' (1) 2024-04-26T04:59:17.787 INFO:journalctl@ceph.mon.a.smithi012.stdout: fs_name foo 2024-04-26T04:59:17.787 INFO:journalctl@ceph.mon.a.smithi012.stdout: epoch 8 2024-04-26T04:59:17.787 INFO:journalctl@ceph.mon.a.smithi012.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-26T04:59:17.787 INFO:journalctl@ceph.mon.a.smithi012.stdout: created 2024-04-26T04:55:07.451848+0000 2024-04-26T04:59:17.787 INFO:journalctl@ceph.mon.a.smithi012.stdout: modified 2024-04-26T04:55:21.872954+0000 2024-04-26T04:59:17.787 INFO:journalctl@ceph.mon.a.smithi012.stdout: tableserver 0 2024-04-26T04:59:17.787 INFO:journalctl@ceph.mon.a.smithi012.stdout: root 0 2024-04-26T04:59:17.787 INFO:journalctl@ceph.mon.a.smithi012.stdout: session_timeout 60 2024-04-26T04:59:17.788 INFO:journalctl@ceph.mon.a.smithi012.stdout: session_autoclose 300 2024-04-26T04:59:17.788 INFO:journalctl@ceph.mon.a.smithi012.stdout: max_file_size 1099511627776 2024-04-26T04:59:17.788 INFO:journalctl@ceph.mon.a.smithi012.stdout: max_xattr_size 65536 2024-04-26T04:59:17.788 INFO:journalctl@ceph.mon.a.smithi012.stdout: required_client_features {} 2024-04-26T04:59:17.788 INFO:journalctl@ceph.mon.a.smithi012.stdout: last_failure 0 2024-04-26T04:59:17.788 INFO:journalctl@ceph.mon.a.smithi012.stdout: last_failure_osd_epoch 0 2024-04-26T04:59:17.788 INFO:journalctl@ceph.mon.a.smithi012.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T04:59:17.788 INFO:journalctl@ceph.mon.a.smithi012.stdout: max_mds 1 2024-04-26T04:59:17.788 INFO:journalctl@ceph.mon.a.smithi012.stdout: in 0 2024-04-26T04:59:17.788 INFO:journalctl@ceph.mon.a.smithi012.stdout: up {0=14664} 2024-04-26T04:59:17.788 INFO:journalctl@ceph.mon.a.smithi012.stdout: failed 2024-04-26T04:59:17.788 INFO:journalctl@ceph.mon.a.smithi012.stdout: damaged 2024-04-26T04:59:17.789 INFO:journalctl@ceph.mon.a.smithi012.stdout: stopped 2024-04-26T04:59:17.789 INFO:journalctl@ceph.mon.a.smithi012.stdout: data_pools [4] 2024-04-26T04:59:17.789 INFO:journalctl@ceph.mon.a.smithi012.stdout: metadata_pool 3 2024-04-26T04:59:17.789 INFO:journalctl@ceph.mon.a.smithi012.stdout: inline_data disabled 2024-04-26T04:59:17.789 INFO:journalctl@ceph.mon.a.smithi012.stdout: balancer 2024-04-26T04:59:17.789 INFO:journalctl@ceph.mon.a.smithi012.stdout: bal_rank_mask -1 2024-04-26T04:59:17.789 INFO:journalctl@ceph.mon.a.smithi012.stdout: standby_count_wanted 1 2024-04-26T04:59:17.789 INFO:journalctl@ceph.mon.a.smithi012.stdout: qdb_cluster leader: 0 members: 2024-04-26T04:59:17.789 INFO:journalctl@ceph.mon.a.smithi012.stdout: [mds.foo.smithi012.fdqwmo{0:14664} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.12:6834/963323745,v1:172.21.15.12:6835/963323745] compat {c=[1],r=[1],i=[7ff]}] 2024-04-26T04:59:17.789 INFO:journalctl@ceph.mon.a.smithi012.stdout: 2024-04-26T04:59:17.789 INFO:journalctl@ceph.mon.a.smithi012.stdout: 2024-04-26T04:59:17.789 INFO:journalctl@ceph.mon.a.smithi012.stdout: Standby daemons: 2024-04-26T04:59:17.789 INFO:journalctl@ceph.mon.a.smithi012.stdout: 2024-04-26T04:59:17.789 INFO:journalctl@ceph.mon.a.smithi012.stdout: [mds.foo.smithi116.kduyef{-1:24520} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.116:6832/529792701,v1:172.21.15.116:6833/529792701] compat {c=[1],r=[1],i=[7ff]}] 2024-04-26T04:59:17.790 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: mon.a@-1(???).osd e79 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-26T04:59:17.790 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: mon.a@-1(???).osd e79 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T04:59:17.790 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: mon.a@-1(???).osd e79 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T04:59:17.790 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: mon.a@-1(???).osd e79 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T04:59:17.790 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-mon[91852]: mon.a@-1(???).paxosservice(auth 1..18) refresh upgraded, format 0 -> 3 2024-04-26T04:59:17.790 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 bash[91833]: db1f873383b097011eb4f61089abb64565ce058ed9342ec7badb8ba4c09eb414 2024-04-26T04:59:17.790 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:17 smithi012 systemd[1]: Started Ceph mon.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:59:18.262 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:17 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:59:17] "GET /metrics HTTP/1.1" 200 33780 "" "Prometheus/2.43.0" 2024-04-26T04:59:18.991 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:59:18 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T04:59:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi116.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:59:18.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:18 smithi116 ceph-mon[29897]: pgmap v22: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:18.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:18 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/4130793863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:18.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:18 smithi116 ceph-mon[29897]: pgmap v23: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:18.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:18 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/2303971893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:18.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:18 smithi116 ceph-mon[29897]: mon.a calling monitor election 2024-04-26T04:59:18.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:18 smithi116 ceph-mon[29897]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:59:18.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:18 smithi116 ceph-mon[29897]: monmap epoch 3 2024-04-26T04:59:18.993 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:18 smithi116 ceph-mon[29897]: fsid 0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T04:59:18.993 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:18 smithi116 ceph-mon[29897]: last_changed 2024-04-26T04:49:56.953431+0000 2024-04-26T04:59:18.993 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:18 smithi116 ceph-mon[29897]: created 2024-04-26T04:48:15.190556+0000 2024-04-26T04:59:18.993 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:18 smithi116 ceph-mon[29897]: min_mon_release 17 (quincy) 2024-04-26T04:59:18.993 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:18 smithi116 ceph-mon[29897]: election_strategy: 1 2024-04-26T04:59:18.993 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:18 smithi116 ceph-mon[29897]: 0: [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] mon.a 2024-04-26T04:59:18.993 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:18 smithi116 ceph-mon[29897]: 1: [v2:172.21.15.116:3300/0,v1:172.21.15.116:6789/0] mon.b 2024-04-26T04:59:18.993 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:18 smithi116 ceph-mon[29897]: 2: [v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0] mon.c 2024-04-26T04:59:18.993 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:18 smithi116 ceph-mon[29897]: fsmap foo:1 {0=foo.smithi012.fdqwmo=up:active} 1 up:standby 2024-04-26T04:59:18.993 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:18 smithi116 ceph-mon[29897]: osdmap e79: 8 total, 8 up, 8 in 2024-04-26T04:59:18.993 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:18 smithi116 ceph-mon[29897]: mgrmap e32: y(active, since 38s), standbys: x 2024-04-26T04:59:18.993 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:18 smithi116 ceph-mon[29897]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T04:59:18.993 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:18 smithi116 ceph-mon[29897]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:59:18.994 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:18 smithi116 ceph-mon[29897]: daemon grafana.a on smithi116 is in unknown state 2024-04-26T04:59:18.994 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:18 smithi116 ceph-mon[29897]: from='mgr.15180 ' entity='' 2024-04-26T04:59:18.994 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:18 smithi116 ceph-mon[29897]: mgrmap e33: y(active, since 38s), standbys: x 2024-04-26T04:59:18.994 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:18 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: ignoring --setuser ceph since I am not root 2024-04-26T04:59:18.994 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:18 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: ignoring --setgroup ceph since I am not root 2024-04-26T04:59:18.994 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:18 smithi116 ceph-mgr[75874]: -- 172.21.15.116:0/4164664378 <== mon.1 v2:172.21.15.116:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5630bf0a25a0 con 0x5630bf077000 2024-04-26T04:59:19.209 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ignoring --setuser ceph since I am not root 2024-04-26T04:59:19.209 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ignoring --setgroup ceph since I am not root 2024-04-26T04:59:19.209 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mgr[82737]: -- 172.21.15.12:0/3333898417 <== mon.2 v2:172.21.15.12:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x563f7a92a5a0 con 0x563f7a8fec00 2024-04-26T04:59:19.209 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:19 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:19.018+0000 7f0e6104b200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:59:19.209 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[91852]: pgmap v22: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:19.210 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/4130793863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:19.210 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[91852]: pgmap v23: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:19.210 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/2303971893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:19.210 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[91852]: mon.a calling monitor election 2024-04-26T04:59:19.210 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[91852]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:59:19.210 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[91852]: monmap epoch 3 2024-04-26T04:59:19.210 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[91852]: fsid 0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T04:59:19.210 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[91852]: last_changed 2024-04-26T04:49:56.953431+0000 2024-04-26T04:59:19.210 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[91852]: created 2024-04-26T04:48:15.190556+0000 2024-04-26T04:59:19.210 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[91852]: min_mon_release 17 (quincy) 2024-04-26T04:59:19.210 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[91852]: election_strategy: 1 2024-04-26T04:59:19.210 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[91852]: 0: [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] mon.a 2024-04-26T04:59:19.210 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[91852]: 1: [v2:172.21.15.116:3300/0,v1:172.21.15.116:6789/0] mon.b 2024-04-26T04:59:19.210 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[91852]: 2: [v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0] mon.c 2024-04-26T04:59:19.211 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[91852]: fsmap foo:1 {0=foo.smithi012.fdqwmo=up:active} 1 up:standby 2024-04-26T04:59:19.211 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[91852]: osdmap e79: 8 total, 8 up, 8 in 2024-04-26T04:59:19.211 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[91852]: mgrmap e32: y(active, since 38s), standbys: x 2024-04-26T04:59:19.211 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[91852]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T04:59:19.211 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[91852]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:59:19.211 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[91852]: daemon grafana.a on smithi116 is in unknown state 2024-04-26T04:59:19.211 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[91852]: from='mgr.15180 ' entity='' 2024-04-26T04:59:19.211 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[91852]: mgrmap e33: y(active, since 38s), standbys: x 2024-04-26T04:59:19.211 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[37567]: pgmap v22: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:19.211 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/4130793863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:19.211 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[37567]: pgmap v23: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:19.211 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/2303971893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:19.211 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[37567]: mon.a calling monitor election 2024-04-26T04:59:19.212 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[37567]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:59:19.212 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[37567]: monmap epoch 3 2024-04-26T04:59:19.212 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[37567]: fsid 0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T04:59:19.212 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[37567]: last_changed 2024-04-26T04:49:56.953431+0000 2024-04-26T04:59:19.212 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[37567]: created 2024-04-26T04:48:15.190556+0000 2024-04-26T04:59:19.212 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[37567]: min_mon_release 17 (quincy) 2024-04-26T04:59:19.212 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[37567]: election_strategy: 1 2024-04-26T04:59:19.212 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[37567]: 0: [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] mon.a 2024-04-26T04:59:19.212 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[37567]: 1: [v2:172.21.15.116:3300/0,v1:172.21.15.116:6789/0] mon.b 2024-04-26T04:59:19.212 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[37567]: 2: [v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0] mon.c 2024-04-26T04:59:19.212 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[37567]: fsmap foo:1 {0=foo.smithi012.fdqwmo=up:active} 1 up:standby 2024-04-26T04:59:19.212 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[37567]: osdmap e79: 8 total, 8 up, 8 in 2024-04-26T04:59:19.212 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[37567]: mgrmap e32: y(active, since 38s), standbys: x 2024-04-26T04:59:19.212 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[37567]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T04:59:19.212 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[37567]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:59:19.213 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[37567]: daemon grafana.a on smithi116 is in unknown state 2024-04-26T04:59:19.213 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[37567]: from='mgr.15180 ' entity='' 2024-04-26T04:59:19.213 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:18 smithi012 ceph-mon[37567]: mgrmap e33: y(active, since 38s), standbys: x 2024-04-26T04:59:19.297 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:19 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:19.014+0000 7f38a3d31200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:59:19.298 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:19 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:19.195+0000 7f38a3d31200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:59:19.298 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:19 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:19.296+0000 7f38a3d31200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:59:19.512 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:19 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:19.207+0000 7f0e6104b200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:59:19.513 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:19 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:19.314+0000 7f0e6104b200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:59:19.991 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:19 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:19.600+0000 7f38a3d31200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:59:19.991 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:19 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:19.684+0000 7f38a3d31200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:59:20.012 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:19 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:19.634+0000 7f0e6104b200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:59:20.013 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:19 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:19.723+0000 7f0e6104b200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:59:20.986 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 1 2024-04-26T04:59:20.991 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:20 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:20.877+0000 7f38a3d31200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:59:21.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:21 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/1507045835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:21.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:21 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/1507045835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:21.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:20 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:20.983+0000 7f0e6104b200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:59:21.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:21 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:21.169+0000 7f0e6104b200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:59:21.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:21 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/1507045835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:59:21.491 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:21.048+0000 7f38a3d31200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:59:21.491 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:21.221+0000 7f38a3d31200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:59:21.652 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:21 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:21.350+0000 7f0e6104b200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:59:21.792 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-26T04:59:21.809 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:21.511+0000 7f38a3d31200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:59:21.810 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:21.604+0000 7f38a3d31200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:59:21.810 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:21 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:21.809+0000 7f38a3d31200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:59:21.969 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:21 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:21.651+0000 7f0e6104b200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:59:21.969 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:21 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:21.749+0000 7f0e6104b200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:59:21.969 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:21 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:21.967+0000 7f0e6104b200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:59:22.067 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:59:22.241 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:22.003+0000 7f38a3d31200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:59:22.262 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:22 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:22.184+0000 7f0e6104b200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:59:22.991 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:22 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:22.601+0000 7f38a3d31200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:59:23.207 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:22 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:22.824+0000 7f0e6104b200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:59:23.345 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:23 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 5. 2024-04-26T04:59:23.345 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:23 smithi116 systemd[1]: Stopped Ceph grafana.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:59:23.345 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:23 smithi116 systemd[1]: Starting Ceph grafana.a for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:59:23.345 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:23 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:23.028+0000 7f38a3d31200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:59:23.346 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:23 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:23.126+0000 7f38a3d31200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:59:23.346 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:23 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:23.344+0000 7f38a3d31200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:59:23.512 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:23 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:23.279+0000 7f0e6104b200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:59:23.512 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:23 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:23.381+0000 7f0e6104b200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:59:23.597 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:23 smithi116 podman[79524]: 2024-04-26 04:59:23.380637909 +0000 UTC m=+0.017231244 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:59:23.597 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:23 smithi116 podman[79524]: 2024-04-26 04:59:23.509250182 +0000 UTC m=+0.145843565 container create 6348a48df674c4885de6cef2667056d75a10a8608fc3239f924b679769152981 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, build-date=2022-04-29T11:48:11.055897, release=239.1651231664, io.openshift.tags=base rhel8, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, io.buildah.version=1.32.0, description=Ceph Grafana Container, distribution-scope=public, vcs-type=git, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner ) 2024-04-26T04:59:23.850 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:23 smithi116 podman[79524]: 2024-04-26 04:59:23.695131982 +0000 UTC m=+0.331725318 container init 6348a48df674c4885de6cef2667056d75a10a8608fc3239f924b679769152981 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, io.buildah.version=1.32.0, io.openshift.expose-services=, architecture=x86_64, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, name=ubi8, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-04-26T04:59:23.850 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:23 smithi116 podman[79524]: 2024-04-26 04:59:23.69781302 +0000 UTC m=+0.334406376 container start 6348a48df674c4885de6cef2667056d75a10a8608fc3239f924b679769152981 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, description=Ceph Grafana Container, com.redhat.component=ubi8-container, io.openshift.expose-services=, io.buildah.version=1.32.0, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., io.openshift.tags=base rhel8, vcs-type=git) 2024-04-26T04:59:23.850 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:23 smithi116 bash[79524]: 6348a48df674c4885de6cef2667056d75a10a8608fc3239f924b679769152981 2024-04-26T04:59:23.850 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:23 smithi116 systemd[1]: Started Ceph grafana.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:59:24.012 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:23 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:23.602+0000 7f0e6104b200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:59:24.165 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:23 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:23.903+0000 7f38a3d31200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:59:24.165 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:23 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:23.998+0000 7f38a3d31200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:59:24.165 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:23 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[79540]: logger=settings t=2024-04-26T04:59:23.849809584Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:59:24.166 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:23 smithi116 podman[79559]: 2024-04-26 04:59:23.875319172 +0000 UTC m=+0.018566388 container died 6348a48df674c4885de6cef2667056d75a10a8608fc3239f924b679769152981 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, io.buildah.version=1.32.0, description=Ceph Grafana Container, architecture=x86_64, com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, version=8.5, distribution-scope=public, io.openshift.expose-services=, vcs-type=git, name=ubi8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-26T04:59:24.487 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:24 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:24.250+0000 7f38a3d31200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:59:24.487 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:24 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:24.386+0000 7f38a3d31200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:59:24.487 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:24 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:24.486+0000 7f38a3d31200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:59:24.512 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:24 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:24.159+0000 7f0e6104b200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:59:24.512 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:24 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:24.258+0000 7f0e6104b200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:59:24.741 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:24 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:24.605+0000 7f38a3d31200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:59:24.741 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:24 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:24.708+0000 7f38a3d31200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:59:24.742 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:24 smithi116 podman[79559]: 2024-04-26 04:59:24.599709463 +0000 UTC m=+0.742956673 container remove 6348a48df674c4885de6cef2667056d75a10a8608fc3239f924b679769152981 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, distribution-scope=public, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container, vendor=Red Hat, Inc., io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , name=ubi8, release=239.1651231664, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-04-26T04:59:24.742 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:24 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:59:24.886 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:24 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:24.517+0000 7f0e6104b200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:59:24.886 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:24 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:24.660+0000 7f0e6104b200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:59:24.886 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:24 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:24.762+0000 7f0e6104b200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:59:24.887 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:24 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:24.885+0000 7f0e6104b200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:59:25.241 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:24 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:59:25.262 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:24 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:24.985+0000 7f0e6104b200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:59:25.823 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:25 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:25.536+0000 7f38a3d31200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:59:25.823 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:25 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:25.757+0000 7f38a3d31200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:59:25.823 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:25 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: [26/Apr/2024:04:59:25] ENGINE Bus STARTING 2024-04-26T04:59:26.077 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:25 smithi012 ceph-mon[91852]: Standby manager daemon x restarted 2024-04-26T04:59:26.078 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:25 smithi012 ceph-mon[91852]: Standby manager daemon x started 2024-04-26T04:59:26.078 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:25 smithi012 ceph-mon[91852]: from='mgr.? 172.21.15.116:0/4045065473' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:59:26.078 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:25 smithi012 ceph-mon[91852]: from='mgr.? 172.21.15.116:0/4045065473' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:59:26.078 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:25 smithi012 ceph-mon[91852]: from='mgr.? 172.21.15.116:0/4045065473' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:59:26.078 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:25 smithi012 ceph-mon[91852]: from='mgr.? 172.21.15.116:0/4045065473' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:59:26.078 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:25 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:25.843+0000 7f0e6104b200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:59:26.079 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:26.076+0000 7f0e6104b200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:59:26.079 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:25 smithi012 ceph-mon[37567]: Standby manager daemon x restarted 2024-04-26T04:59:26.079 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:25 smithi012 ceph-mon[37567]: Standby manager daemon x started 2024-04-26T04:59:26.079 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:25 smithi012 ceph-mon[37567]: from='mgr.? 172.21.15.116:0/4045065473' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:59:26.079 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:25 smithi012 ceph-mon[37567]: from='mgr.? 172.21.15.116:0/4045065473' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:59:26.079 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:25 smithi012 ceph-mon[37567]: from='mgr.? 172.21.15.116:0/4045065473' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:59:26.079 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:25 smithi012 ceph-mon[37567]: from='mgr.? 172.21.15.116:0/4045065473' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:59:26.230 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:25 smithi116 ceph-mon[29897]: Standby manager daemon x restarted 2024-04-26T04:59:26.230 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:25 smithi116 ceph-mon[29897]: Standby manager daemon x started 2024-04-26T04:59:26.230 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:25 smithi116 ceph-mon[29897]: from='mgr.? 172.21.15.116:0/4045065473' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:59:26.230 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:25 smithi116 ceph-mon[29897]: from='mgr.? 172.21.15.116:0/4045065473' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:59:26.230 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:25 smithi116 ceph-mon[29897]: from='mgr.? 172.21.15.116:0/4045065473' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:59:26.231 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:25 smithi116 ceph-mon[29897]: from='mgr.? 172.21.15.116:0/4045065473' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:59:26.231 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:25 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: CherryPy Checker: 2024-04-26T04:59:26.231 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:25 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: The Application mounted at '' has an empty config. 2024-04-26T04:59:26.231 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:25 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: 2024-04-26T04:59:26.231 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:25 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: [26/Apr/2024:04:59:25] ENGINE Serving on http://:::9283 2024-04-26T04:59:26.231 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 04:59:25 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x[75870]: [26/Apr/2024:04:59:25] ENGINE Bus STARTED 2024-04-26T04:59:26.513 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: [26/Apr/2024:04:59:26] ENGINE Bus STARTING 2024-04-26T04:59:26.513 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:26.203+0000 7f0dc8ba0700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:59:26.513 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:26.203+0000 7f0dc8ba0700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:59:26.513 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:26.203+0000 7f0dc8ba0700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:59:26.514 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:26.203+0000 7f0dc8ba0700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:59:26.514 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:26.203+0000 7f0dc8ba0700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:59:26.514 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:26.204+0000 7f0dc4b98700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:59:26.514 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:26.204+0000 7f0dc4b98700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:59:26.514 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:26.204+0000 7f0dc4b98700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:59:26.514 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:26.204+0000 7f0dc4b98700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:59:26.514 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:26.204+0000 7f0dc4b98700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:59:26.514 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: CherryPy Checker: 2024-04-26T04:59:26.514 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: The Application mounted at '' has an empty config. 2024-04-26T04:59:26.514 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:26.514 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: [26/Apr/2024:04:59:26] ENGINE Serving on http://:::9283 2024-04-26T04:59:26.515 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: [26/Apr/2024:04:59:26] ENGINE Bus STARTED 2024-04-26T04:59:27.095 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[91852]: mgrmap e34: y(active, since 45s), standbys: x 2024-04-26T04:59:27.095 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[91852]: Active manager daemon y restarted 2024-04-26T04:59:27.095 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[91852]: Activating manager daemon y 2024-04-26T04:59:27.095 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[91852]: osdmap e80: 8 total, 8 up, 8 in 2024-04-26T04:59:27.095 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[91852]: mgrmap e35: y(active, starting, since 0.00591926s), standbys: x 2024-04-26T04:59:27.095 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:59:27.095 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:59:27.095 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:59:27.095 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi012.fdqwmo"}]: dispatch 2024-04-26T04:59:27.095 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi116.kduyef"}]: dispatch 2024-04-26T04:59:27.095 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:59:27.095 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:59:27.096 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:59:27.096 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:59:27.096 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:59:27.096 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:59:27.096 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:59:27.096 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:59:27.096 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:59:27.096 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:59:27.096 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:59:27.096 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:59:27.096 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:59:27.096 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[91852]: Manager daemon y is now available 2024-04-26T04:59:27.096 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:27.096 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:27.096 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:59:27.096 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:59:27.097 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[37567]: mgrmap e34: y(active, since 45s), standbys: x 2024-04-26T04:59:27.097 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[37567]: Active manager daemon y restarted 2024-04-26T04:59:27.097 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[37567]: Activating manager daemon y 2024-04-26T04:59:27.097 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[37567]: osdmap e80: 8 total, 8 up, 8 in 2024-04-26T04:59:27.097 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[37567]: mgrmap e35: y(active, starting, since 0.00591926s), standbys: x 2024-04-26T04:59:27.097 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:59:27.097 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:59:27.097 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:59:27.097 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi012.fdqwmo"}]: dispatch 2024-04-26T04:59:27.097 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi116.kduyef"}]: dispatch 2024-04-26T04:59:27.097 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:59:27.097 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:59:27.097 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:59:27.097 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:59:27.098 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:59:27.098 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:59:27.098 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:59:27.098 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:59:27.098 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:59:27.098 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:59:27.098 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:59:27.098 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:59:27.098 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:59:27.098 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[37567]: Manager daemon y is now available 2024-04-26T04:59:27.098 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:27.098 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:27.098 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:59:27.098 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:26 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:59:27.106 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T04:59:27.106 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.a smithi012 *:9093,9094 running (84s) 42s ago 5m 24.7M - 0.25.0 c8568f914cd2 217bb19e153b 2024-04-26T04:59:27.106 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.a smithi116 *:3000 unknown 20s ago 5m - - 2024-04-26T04:59:27.106 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foo.smithi012.fdqwmo smithi012 running (4m) 42s ago 4m 15.3M - 17.2.7-902-gc61472c8 0de407c1b3af 53f6c82f01b4 2024-04-26T04:59:27.106 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foo.smithi116.kduyef smithi116 running (4m) 20s ago 4m 14.9M - 17.2.7-902-gc61472c8 0de407c1b3af 4c0a0b911593 2024-04-26T04:59:27.106 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.x smithi116 *:8443,9283,8765 running (25s) 20s ago 9m 70.1M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 5499bf989999 2024-04-26T04:59:27.106 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.y smithi012 *:8443,9283,8765 running (55s) 42s ago 11m 472M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 3a46d0031772 2024-04-26T04:59:27.106 INFO:teuthology.orchestra.run.smithi012.stdout:mon.a smithi012 running (11m) 42s ago 11m 50.6M 2048M 17.2.7-902-gc61472c8 0de407c1b3af 1b1568ad49e7 2024-04-26T04:59:27.106 INFO:teuthology.orchestra.run.smithi012.stdout:mon.b smithi116 running (9m) 20s ago 9m 44.1M 2048M 17.2.7-902-gc61472c8 0de407c1b3af e6cd16f01417 2024-04-26T04:59:27.106 INFO:teuthology.orchestra.run.smithi012.stdout:mon.c smithi012 running (9m) 42s ago 9m 41.7M 2048M 17.2.7-902-gc61472c8 0de407c1b3af 0f03d4c16528 2024-04-26T04:59:27.106 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.a smithi012 *:9100 running (81s) 42s ago 4m 8744k - 1.5.0 0da6a335fe13 6390338999c1 2024-04-26T04:59:27.106 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.b smithi116 *:9100 running (61s) 20s ago 4m 14.2M - 1.5.0 0da6a335fe13 7d0b1e809531 2024-04-26T04:59:27.106 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (8m) 42s ago 8m 47.3M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 25b6843697db 2024-04-26T04:59:27.107 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (8m) 42s ago 8m 45.7M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 8117a7da3ae3 2024-04-26T04:59:27.107 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (8m) 42s ago 8m 52.4M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 432af63060b8 2024-04-26T04:59:27.107 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (7m) 42s ago 7m 46.0M 2481M 17.2.7-902-gc61472c8 0de407c1b3af f65d4289a970 2024-04-26T04:59:27.107 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi116 running (7m) 20s ago 7m 45.7M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 6b0bbf77cb39 2024-04-26T04:59:27.107 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi116 running (6m) 20s ago 6m 51.1M 2480M 17.2.7-902-gc61472c8 0de407c1b3af a551ef15fa7e 2024-04-26T04:59:27.107 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi116 running (6m) 20s ago 6m 50.1M 2480M 17.2.7-902-gc61472c8 0de407c1b3af cdc0eb6f4fa2 2024-04-26T04:59:27.107 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi116 running (6m) 20s ago 6m 50.5M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 3f6689213fda 2024-04-26T04:59:27.107 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.a smithi116 *:9095 running (31s) 20s ago 5m 34.5M - 2.43.0 a07b618ecd1d e9e5b93523ae 2024-04-26T04:59:27.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:26 smithi116 ceph-mon[29897]: mgrmap e34: y(active, since 45s), standbys: x 2024-04-26T04:59:27.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:26 smithi116 ceph-mon[29897]: Active manager daemon y restarted 2024-04-26T04:59:27.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:26 smithi116 ceph-mon[29897]: Activating manager daemon y 2024-04-26T04:59:27.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:26 smithi116 ceph-mon[29897]: osdmap e80: 8 total, 8 up, 8 in 2024-04-26T04:59:27.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:26 smithi116 ceph-mon[29897]: mgrmap e35: y(active, starting, since 0.00591926s), standbys: x 2024-04-26T04:59:27.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:26 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:59:27.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:26 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:59:27.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:26 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:59:27.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:26 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi012.fdqwmo"}]: dispatch 2024-04-26T04:59:27.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:26 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi116.kduyef"}]: dispatch 2024-04-26T04:59:27.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:26 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:59:27.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:26 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:59:27.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:26 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:59:27.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:26 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:59:27.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:26 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:59:27.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:26 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:59:27.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:26 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:59:27.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:26 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:59:27.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:26 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:59:27.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:26 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:59:27.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:26 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:59:27.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:26 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:59:27.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:26 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:59:27.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:26 smithi116 ceph-mon[29897]: Manager daemon y is now available 2024-04-26T04:59:27.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:26 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:27.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:26 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:27.244 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:26 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:59:27.244 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:26 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:59:27.431 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:27 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T04:59:27.092+0000 7f0e1e4f1700 -1 mgr.server handle_report got status from non-daemon mon.a 2024-04-26T04:59:27.842 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade pause' 2024-04-26T04:59:28.190 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:27 smithi116 ceph-mon[29897]: [26/Apr/2024:04:59:26] ENGINE Bus STARTING 2024-04-26T04:59:28.190 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:27 smithi116 ceph-mon[29897]: [26/Apr/2024:04:59:26] ENGINE Serving on http://172.21.15.12:8765 2024-04-26T04:59:28.190 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:27 smithi116 ceph-mon[29897]: [26/Apr/2024:04:59:26] ENGINE Serving on https://172.21.15.12:7150 2024-04-26T04:59:28.190 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:27 smithi116 ceph-mon[29897]: [26/Apr/2024:04:59:26] ENGINE Bus STARTED 2024-04-26T04:59:28.190 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:27 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:28.190 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:27 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:28.190 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:27 smithi116 ceph-mon[29897]: mgrmap e36: y(active, since 1.0113s), standbys: x 2024-04-26T04:59:28.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:27 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:59:27] "GET /metrics HTTP/1.1" 200 31141 "" "Prometheus/2.43.0" 2024-04-26T04:59:28.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:27 smithi012 ceph-mon[91852]: [26/Apr/2024:04:59:26] ENGINE Bus STARTING 2024-04-26T04:59:28.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:27 smithi012 ceph-mon[91852]: [26/Apr/2024:04:59:26] ENGINE Serving on http://172.21.15.12:8765 2024-04-26T04:59:28.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:27 smithi012 ceph-mon[91852]: [26/Apr/2024:04:59:26] ENGINE Serving on https://172.21.15.12:7150 2024-04-26T04:59:28.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:27 smithi012 ceph-mon[91852]: [26/Apr/2024:04:59:26] ENGINE Bus STARTED 2024-04-26T04:59:28.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:27 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:28.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:27 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:28.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:27 smithi012 ceph-mon[91852]: mgrmap e36: y(active, since 1.0113s), standbys: x 2024-04-26T04:59:28.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:27 smithi012 ceph-mon[37567]: [26/Apr/2024:04:59:26] ENGINE Bus STARTING 2024-04-26T04:59:28.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:27 smithi012 ceph-mon[37567]: [26/Apr/2024:04:59:26] ENGINE Serving on http://172.21.15.12:8765 2024-04-26T04:59:28.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:27 smithi012 ceph-mon[37567]: [26/Apr/2024:04:59:26] ENGINE Serving on https://172.21.15.12:7150 2024-04-26T04:59:28.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:27 smithi012 ceph-mon[37567]: [26/Apr/2024:04:59:26] ENGINE Bus STARTED 2024-04-26T04:59:28.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:27 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:28.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:27 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:28.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:27 smithi012 ceph-mon[37567]: mgrmap e36: y(active, since 1.0113s), standbys: x 2024-04-26T04:59:28.749 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:59:28.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:28 smithi116 ceph-mon[29897]: from='client.25259 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:59:28.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:28 smithi116 ceph-mon[29897]: pgmap v3: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:28.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:28 smithi116 ceph-mon[29897]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T04:59:28.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:28 smithi116 ceph-mon[29897]: Cluster is now healthy 2024-04-26T04:59:28.992 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:59:28 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T04:59:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi116.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:59:29.097 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:28 smithi012 ceph-mon[91852]: from='client.25259 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:59:29.097 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:28 smithi012 ceph-mon[91852]: pgmap v3: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:29.097 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:28 smithi012 ceph-mon[91852]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T04:59:29.097 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:28 smithi012 ceph-mon[91852]: Cluster is now healthy 2024-04-26T04:59:29.097 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:28 smithi012 ceph-mon[37567]: from='client.25259 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:59:29.097 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:28 smithi012 ceph-mon[37567]: pgmap v3: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:29.097 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:28 smithi012 ceph-mon[37567]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T04:59:29.097 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:28 smithi012 ceph-mon[37567]: Cluster is now healthy 2024-04-26T04:59:30.122 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:29 smithi116 ceph-mon[29897]: pgmap v4: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:30.122 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:29 smithi116 ceph-mon[29897]: mgrmap e37: y(active, since 2s), standbys: x 2024-04-26T04:59:30.214 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:29 smithi012 ceph-mon[91852]: pgmap v4: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:30.214 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:29 smithi012 ceph-mon[91852]: mgrmap e37: y(active, since 2s), standbys: x 2024-04-26T04:59:30.214 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:29 smithi012 ceph-mon[37567]: pgmap v4: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:30.214 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:29 smithi012 ceph-mon[37567]: mgrmap e37: y(active, since 2s), standbys: x 2024-04-26T04:59:31.350 INFO:teuthology.orchestra.run.smithi012.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:59:31.607 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:31 smithi012 ceph-mon[91852]: pgmap v5: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:31.607 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:31 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:31.607 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:31 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:31.607 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:31 smithi012 ceph-mon[91852]: mgrmap e38: y(active, since 4s), standbys: x 2024-04-26T04:59:31.608 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:31 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:31.608 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:31 smithi012 ceph-mon[37567]: pgmap v5: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:31.608 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:31 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:31.608 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:31 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:31.608 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:31 smithi012 ceph-mon[37567]: mgrmap e38: y(active, since 4s), standbys: x 2024-04-26T04:59:31.608 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:31 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:31.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:31 smithi116 ceph-mon[29897]: pgmap v5: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:31.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:31 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:31.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:31 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:31.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:31 smithi116 ceph-mon[29897]: mgrmap e38: y(active, since 4s), standbys: x 2024-04-26T04:59:31.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:31 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:32.245 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'sleep 60' 2024-04-26T04:59:32.607 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:59:32.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:32 smithi116 ceph-mon[29897]: from='client.25301 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:59:32.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:32 smithi116 ceph-mon[29897]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:59:32.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:32 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:32.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:32 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:32.794 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:32 smithi012 ceph-mon[91852]: from='client.25301 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:59:32.794 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:32 smithi012 ceph-mon[91852]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:59:32.794 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:32 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:32.795 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:32 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:32.795 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:32 smithi012 ceph-mon[37567]: from='client.25301 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:59:32.795 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:32 smithi012 ceph-mon[37567]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:59:32.795 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:32 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:32.795 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:32 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:33.642 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:33 smithi116 ceph-mon[29897]: pgmap v6: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:33.642 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:33 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:33.642 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:33 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:33.681 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:33 smithi012 ceph-mon[91852]: pgmap v6: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:33.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:33 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:33.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:33 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:33.682 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:33 smithi012 ceph-mon[37567]: pgmap v6: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:33.682 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:33 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:33.682 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:33 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:35.214 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:34 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 6. 2024-04-26T04:59:35.214 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:34 smithi116 systemd[1]: Stopped Ceph grafana.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:59:35.214 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:34 smithi116 systemd[1]: Starting Ceph grafana.a for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:59:35.214 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:35 smithi116 podman[83034]: 2024-04-26 04:59:35.092167212 +0000 UTC m=+0.017691628 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:59:35.214 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:35 smithi116 podman[83034]: 2024-04-26 04:59:35.213776593 +0000 UTC m=+0.139300999 container create a471c55bcf014bddda4143d1d67a2ac31c0d47458cb9e1f5b9a4b451d46e0af6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, version=8.5, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, distribution-scope=public, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, maintainer=Paul Cuzner , name=ubi8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-04-26T04:59:35.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:35 smithi116 ceph-mon[29897]: pgmap v7: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T04:59:35.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:35 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:35.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:35 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:35.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:35 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:35.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:35 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:35.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:35 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:59:35.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:35 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:59:35.992 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:35 smithi116 podman[83034]: 2024-04-26 04:59:35.522222854 +0000 UTC m=+0.447747263 container init a471c55bcf014bddda4143d1d67a2ac31c0d47458cb9e1f5b9a4b451d46e0af6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., description=Ceph Grafana Container, distribution-scope=public, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-04-29T11:48:11.055897, release=239.1651231664, io.buildah.version=1.32.0, version=8.5, vcs-type=git, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , name=ubi8, io.openshift.expose-services=, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container) 2024-04-26T04:59:35.992 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:35 smithi116 podman[83034]: 2024-04-26 04:59:35.524622854 +0000 UTC m=+0.450147270 container start a471c55bcf014bddda4143d1d67a2ac31c0d47458cb9e1f5b9a4b451d46e0af6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, maintainer=Paul Cuzner , name=ubi8, release=239.1651231664, version=8.5, build-date=2022-04-29T11:48:11.055897, vcs-type=git, com.redhat.component=ubi8-container, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, io.openshift.tags=base rhel8, description=Ceph Grafana Container, architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-04-26T04:59:35.992 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:35 smithi116 bash[83034]: a471c55bcf014bddda4143d1d67a2ac31c0d47458cb9e1f5b9a4b451d46e0af6 2024-04-26T04:59:35.992 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:35 smithi116 systemd[1]: Started Ceph grafana.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:59:35.992 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:35 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[83281]: logger=settings t=2024-04-26T04:59:35.678921215Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:59:35.993 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:35 smithi116 podman[83608]: 2024-04-26 04:59:35.705586841 +0000 UTC m=+0.019323611 container died a471c55bcf014bddda4143d1d67a2ac31c0d47458cb9e1f5b9a4b451d46e0af6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, release=239.1651231664, version=8.5, description=Ceph Grafana Container, com.redhat.component=ubi8-container, architecture=x86_64, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, name=ubi8, distribution-scope=public) 2024-04-26T04:59:36.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:35 smithi012 ceph-mon[91852]: pgmap v7: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T04:59:36.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:35 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:36.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:35 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:36.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:35 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:36.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:35 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:36.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:35 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:59:36.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:35 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:59:36.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:35 smithi012 ceph-mon[37567]: pgmap v7: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T04:59:36.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:35 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:36.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:35 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:36.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:35 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:36.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:35 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:36.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:35 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:59:36.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:35 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:59:36.654 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:36 smithi116 podman[83608]: 2024-04-26 04:59:36.338633343 +0000 UTC m=+0.652370142 container remove a471c55bcf014bddda4143d1d67a2ac31c0d47458cb9e1f5b9a4b451d46e0af6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, description=Ceph Grafana Container, io.openshift.expose-services=, release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, build-date=2022-04-29T11:48:11.055897, io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, vcs-type=git, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, distribution-scope=public, version=8.5, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-04-26T04:59:36.655 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:36 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:59:36.655 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:36 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:59:36.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:36 smithi116 ceph-mon[29897]: Updating smithi012:/etc/ceph/ceph.conf 2024-04-26T04:59:36.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:36 smithi116 ceph-mon[29897]: Updating smithi116:/etc/ceph/ceph.conf 2024-04-26T04:59:36.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:36 smithi116 ceph-mon[29897]: Updating smithi116:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:59:36.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:36 smithi116 ceph-mon[29897]: Updating smithi012:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:59:36.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:36 smithi116 ceph-mon[29897]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:59:36.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:36 smithi116 ceph-mon[29897]: Updating smithi116:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:59:36.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:36 smithi116 ceph-mon[29897]: Updating smithi012:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:59:36.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:36 smithi116 ceph-mon[29897]: Updating smithi116:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:59:36.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:36 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:36.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:36 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:36.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:36 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:36.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:36 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:36.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:36 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:36.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:36 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:36.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:36 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:59:36.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:36 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:59:36.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:36 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:37.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:36 smithi012 ceph-mon[37567]: Updating smithi012:/etc/ceph/ceph.conf 2024-04-26T04:59:37.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:36 smithi012 ceph-mon[37567]: Updating smithi116:/etc/ceph/ceph.conf 2024-04-26T04:59:37.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:36 smithi012 ceph-mon[37567]: Updating smithi116:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:59:37.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:36 smithi012 ceph-mon[37567]: Updating smithi012:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:59:37.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:36 smithi012 ceph-mon[37567]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:59:37.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:36 smithi012 ceph-mon[37567]: Updating smithi116:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:59:37.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:36 smithi012 ceph-mon[37567]: Updating smithi012:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:59:37.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:36 smithi012 ceph-mon[37567]: Updating smithi116:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:59:37.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:36 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:37.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:36 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:37.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:36 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:37.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:36 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:37.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:36 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:37.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:36 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:37.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:36 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:59:37.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:36 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:59:37.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:36 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:37.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:36 smithi012 ceph-mon[91852]: Updating smithi012:/etc/ceph/ceph.conf 2024-04-26T04:59:37.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:36 smithi012 ceph-mon[91852]: Updating smithi116:/etc/ceph/ceph.conf 2024-04-26T04:59:37.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:36 smithi012 ceph-mon[91852]: Updating smithi116:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:59:37.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:36 smithi012 ceph-mon[91852]: Updating smithi012:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:59:37.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:36 smithi012 ceph-mon[91852]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:59:37.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:36 smithi012 ceph-mon[91852]: Updating smithi116:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:59:37.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:36 smithi012 ceph-mon[91852]: Updating smithi012:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:59:37.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:36 smithi012 ceph-mon[91852]: Updating smithi116:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:59:37.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:36 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:37.015 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:36 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:37.015 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:36 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:37.015 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:36 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:37.015 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:36 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:37.015 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:36 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:37.015 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:36 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:59:37.015 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:36 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:59:37.015 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:36 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T04:59:37.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:37 smithi116 ceph-mon[29897]: pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:59:37.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:37 smithi116 ceph-mon[29897]: pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:59:37.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:37 smithi116 ceph-mon[29897]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:59:37.992 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:37 smithi012 ceph-mon[91852]: pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:59:37.992 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:37 smithi012 ceph-mon[91852]: pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:59:37.992 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:37 smithi012 ceph-mon[91852]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:59:37.992 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:37 smithi012 ceph-mon[37567]: pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:59:37.993 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:37 smithi012 ceph-mon[37567]: pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:59:37.993 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:37 smithi012 ceph-mon[37567]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:59:38.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:37 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:59:37] "GET /metrics HTTP/1.1" 200 31141 "" "Prometheus/2.43.0" 2024-04-26T04:59:38.991 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:59:38 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T04:59:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi116.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:59:39.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:39 smithi116 ceph-mon[29897]: pgmap v10: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:59:40.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:39 smithi012 ceph-mon[91852]: pgmap v10: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:59:40.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:39 smithi012 ceph-mon[37567]: pgmap v10: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:59:41.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:41 smithi116 ceph-mon[29897]: pgmap v11: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:59:41.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:41 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:42.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:41 smithi012 ceph-mon[91852]: pgmap v11: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:59:42.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:41 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:42.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:41 smithi012 ceph-mon[37567]: pgmap v11: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:59:42.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:41 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:43.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:43 smithi116 ceph-mon[29897]: pgmap v12: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:59:44.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:43 smithi012 ceph-mon[91852]: pgmap v12: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:59:44.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:43 smithi012 ceph-mon[37567]: pgmap v12: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-26T04:59:45.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:45 smithi116 ceph-mon[29897]: pgmap v13: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:46.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:45 smithi012 ceph-mon[91852]: pgmap v13: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:46.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:45 smithi012 ceph-mon[37567]: pgmap v13: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:47.114 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:46 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 7. 2024-04-26T04:59:47.114 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:46 smithi116 systemd[1]: Stopped Ceph grafana.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:59:47.114 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:46 smithi116 systemd[1]: Starting Ceph grafana.a for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:59:47.114 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:46 smithi116 podman[84013]: 2024-04-26 04:59:46.877940523 +0000 UTC m=+0.018264849 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:59:47.114 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:46 smithi116 podman[84013]: 2024-04-26 04:59:46.989358121 +0000 UTC m=+0.129682435 container create ca211250d002a000e247db54a59653eec97a91894e079414563af548396546c3 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, release=239.1651231664, io.openshift.expose-services=, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, distribution-scope=public, com.redhat.component=ubi8-container, maintainer=Paul Cuzner , io.openshift.tags=base rhel8, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, vcs-type=git, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-04-26T04:59:47.411 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:47 smithi116 podman[84013]: 2024-04-26 04:59:47.258357308 +0000 UTC m=+0.398681621 container init ca211250d002a000e247db54a59653eec97a91894e079414563af548396546c3 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, io.buildah.version=1.32.0, description=Ceph Grafana Container, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64, vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, io.openshift.expose-services=, com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, vendor=Red Hat, Inc., version=8.5, distribution-scope=public) 2024-04-26T04:59:47.411 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:47 smithi116 podman[84013]: 2024-04-26 04:59:47.260871041 +0000 UTC m=+0.401195355 container start ca211250d002a000e247db54a59653eec97a91894e079414563af548396546c3 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, version=8.5, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, distribution-scope=public, release=239.1651231664, io.buildah.version=1.32.0, io.openshift.expose-services=, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, vendor=Red Hat, Inc.) 2024-04-26T04:59:47.411 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:47 smithi116 bash[84013]: ca211250d002a000e247db54a59653eec97a91894e079414563af548396546c3 2024-04-26T04:59:47.411 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:47 smithi116 systemd[1]: Started Ceph grafana.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:59:47.669 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:47 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[84028]: logger=settings t=2024-04-26T04:59:47.410330343Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:59:47.669 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:47 smithi116 podman[84046]: 2024-04-26 04:59:47.442478701 +0000 UTC m=+0.019366436 container died ca211250d002a000e247db54a59653eec97a91894e079414563af548396546c3 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, name=ubi8, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-04-26T04:59:47.991 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:47 smithi012 ceph-mon[91852]: pgmap v14: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:47.991 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:47 smithi012 ceph-mon[37567]: pgmap v14: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:47.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:47 smithi116 ceph-mon[29897]: pgmap v14: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:48.262 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:47 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:59:47] "GET /metrics HTTP/1.1" 200 33777 "" "Prometheus/2.43.0" 2024-04-26T04:59:48.297 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:48 smithi116 podman[84046]: 2024-04-26 04:59:48.037647541 +0000 UTC m=+0.614535286 container remove ca211250d002a000e247db54a59653eec97a91894e079414563af548396546c3 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, name=ubi8, io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, build-date=2022-04-29T11:48:11.055897) 2024-04-26T04:59:48.297 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:48 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:59:48.556 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:48 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:59:48.991 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:59:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T04:59:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi116.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:59:49.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:49 smithi116 ceph-mon[29897]: pgmap v15: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:50.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:49 smithi012 ceph-mon[91852]: pgmap v15: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:50.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:49 smithi012 ceph-mon[37567]: pgmap v15: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:51.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:51 smithi116 ceph-mon[29897]: pgmap v16: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:52.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:51 smithi012 ceph-mon[91852]: pgmap v16: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:52.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:51 smithi012 ceph-mon[37567]: pgmap v16: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:53.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:53 smithi116 ceph-mon[29897]: pgmap v17: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:54.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:53 smithi012 ceph-mon[91852]: pgmap v17: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:54.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:53 smithi012 ceph-mon[37567]: pgmap v17: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:55.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:55 smithi116 ceph-mon[29897]: pgmap v18: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:56.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:55 smithi012 ceph-mon[91852]: pgmap v18: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:56.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:55 smithi012 ceph-mon[37567]: pgmap v18: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:56.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:56 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:57.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:56 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:57.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:56 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:57.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:57 smithi116 ceph-mon[29897]: pgmap v19: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:57.991 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:57 smithi012 ceph-mon[91852]: pgmap v19: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:57.992 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:57 smithi012 ceph-mon[37567]: pgmap v19: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:58.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 04:59:57 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:04:59:57] "GET /metrics HTTP/1.1" 200 33776 "" "Prometheus/2.43.0" 2024-04-26T04:59:58.855 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 04:59:58 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T04:59:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi116.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:59:58.855 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:58 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 8. 2024-04-26T04:59:58.855 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:58 smithi116 systemd[1]: Stopped Ceph grafana.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:59:58.855 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:58 smithi116 systemd[1]: Starting Ceph grafana.a for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T04:59:58.855 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:58 smithi116 podman[84154]: 2024-04-26 04:59:58.632518908 +0000 UTC m=+0.018603433 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:59:58.855 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:58 smithi116 podman[84154]: 2024-04-26 04:59:58.738043577 +0000 UTC m=+0.124128117 container create 16ca66a327325931a1b1b25ee09b0c2a64b3b84e1e6078a5cab504aff77520b8 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, vcs-type=git, io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.32.0, version=8.5, architecture=x86_64, com.redhat.component=ubi8-container, distribution-scope=public, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, maintainer=Paul Cuzner , io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, build-date=2022-04-29T11:48:11.055897) 2024-04-26T04:59:59.411 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:58 smithi116 podman[84154]: 2024-04-26 04:59:58.998520074 +0000 UTC m=+0.384604581 container init 16ca66a327325931a1b1b25ee09b0c2a64b3b84e1e6078a5cab504aff77520b8 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, vcs-type=git, io.buildah.version=1.32.0, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, distribution-scope=public, com.redhat.component=ubi8-container, build-date=2022-04-29T11:48:11.055897, architecture=x86_64, io.openshift.expose-services=) 2024-04-26T04:59:59.411 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:59 smithi116 podman[84154]: 2024-04-26 04:59:59.000995329 +0000 UTC m=+0.387079840 container start 16ca66a327325931a1b1b25ee09b0c2a64b3b84e1e6078a5cab504aff77520b8 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, release=239.1651231664, version=8.5, io.buildah.version=1.32.0, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , description=Ceph Grafana Container, vcs-type=git, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.tags=base rhel8, io.openshift.expose-services=, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-04-29T11:48:11.055897, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-04-26T04:59:59.411 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:59 smithi116 bash[84154]: 16ca66a327325931a1b1b25ee09b0c2a64b3b84e1e6078a5cab504aff77520b8 2024-04-26T04:59:59.411 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:59 smithi116 systemd[1]: Started Ceph grafana.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T04:59:59.411 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:59 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a[84169]: logger=settings t=2024-04-26T04:59:59.146552679Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:59:59.411 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:59 smithi116 podman[84187]: 2024-04-26 04:59:59.175666117 +0000 UTC m=+0.018431802 container died 16ca66a327325931a1b1b25ee09b0c2a64b3b84e1e6078a5cab504aff77520b8 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, io.openshift.tags=base rhel8, name=ubi8, maintainer=Paul Cuzner , summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, distribution-scope=public, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vendor=Red Hat, Inc., release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-04-26T04:59:59.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 04:59:59 smithi116 ceph-mon[29897]: pgmap v20: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:59.992 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:59 smithi116 podman[84187]: 2024-04-26 04:59:59.693971474 +0000 UTC m=+0.536737141 container remove 16ca66a327325931a1b1b25ee09b0c2a64b3b84e1e6078a5cab504aff77520b8 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, vcs-type=git, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, distribution-scope=public, build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-04-26T04:59:59.992 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:59 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:59:59.992 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 04:59:59 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T05:00:00.015 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 04:59:59 smithi012 ceph-mon[91852]: pgmap v20: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:00.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 04:59:59 smithi012 ceph-mon[37567]: pgmap v20: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:00.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:00 smithi116 ceph-mon[29897]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T05:00:01.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:00 smithi012 ceph-mon[91852]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T05:00:01.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:00 smithi012 ceph-mon[37567]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T05:00:01.951 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:01 smithi116 ceph-mon[29897]: pgmap v21: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:02.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:01 smithi012 ceph-mon[91852]: pgmap v21: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:02.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:01 smithi012 ceph-mon[37567]: pgmap v21: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:03.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:03 smithi116 ceph-mon[29897]: pgmap v22: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:04.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:03 smithi012 ceph-mon[91852]: pgmap v22: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:04.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:03 smithi012 ceph-mon[37567]: pgmap v22: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:05.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:05 smithi116 ceph-mon[29897]: pgmap v23: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:06.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:05 smithi012 ceph-mon[91852]: pgmap v23: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:06.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:05 smithi012 ceph-mon[37567]: pgmap v23: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:07.990 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:07 smithi012 ceph-mon[91852]: pgmap v24: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:07.991 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:07 smithi012 ceph-mon[37567]: pgmap v24: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:07.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:07 smithi116 ceph-mon[29897]: pgmap v24: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:08.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:00:07 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:00:07] "GET /metrics HTTP/1.1" 200 33776 "" "Prometheus/2.43.0" 2024-04-26T05:00:08.991 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:00:08 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:00:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi116.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:00:09.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:09 smithi116 ceph-mon[29897]: pgmap v25: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:10.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:09 smithi012 ceph-mon[91852]: pgmap v25: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:10.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:09 smithi012 ceph-mon[37567]: pgmap v25: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:10.491 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 05:00:09 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 9. 2024-04-26T05:00:10.491 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 05:00:09 smithi116 systemd[1]: Stopped Ceph grafana.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T05:00:10.491 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 05:00:09 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@grafana.a.service: Start request repeated too quickly. 2024-04-26T05:00:10.492 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 05:00:09 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T05:00:10.492 INFO:journalctl@ceph.grafana.a.smithi116.stdout:Apr 26 05:00:09 smithi116 systemd[1]: Failed to start Ceph grafana.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T05:00:11.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:11 smithi116 ceph-mon[29897]: pgmap v26: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:11.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:11 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:12.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:11 smithi012 ceph-mon[91852]: pgmap v26: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:12.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:11 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:12.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:11 smithi012 ceph-mon[37567]: pgmap v26: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:12.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:11 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:13.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:13 smithi116 ceph-mon[29897]: pgmap v27: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:14.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:13 smithi012 ceph-mon[91852]: pgmap v27: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:14.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:13 smithi012 ceph-mon[37567]: pgmap v27: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:15.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:15 smithi116 ceph-mon[29897]: pgmap v28: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:16.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:15 smithi012 ceph-mon[91852]: pgmap v28: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:16.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:15 smithi012 ceph-mon[37567]: pgmap v28: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:17.991 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:17 smithi012 ceph-mon[91852]: pgmap v29: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:17.992 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:17 smithi012 ceph-mon[37567]: pgmap v29: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:17.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:17 smithi116 ceph-mon[29897]: pgmap v29: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:18.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:00:17 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:00:17] "GET /metrics HTTP/1.1" 200 33774 "" "Prometheus/2.43.0" 2024-04-26T05:00:18.991 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:00:18 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:00:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi116.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:00:19.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:19 smithi116 ceph-mon[29897]: pgmap v30: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:20.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:19 smithi012 ceph-mon[91852]: pgmap v30: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:20.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:19 smithi012 ceph-mon[37567]: pgmap v30: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:21.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:21 smithi116 ceph-mon[29897]: pgmap v31: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:22.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:21 smithi012 ceph-mon[91852]: pgmap v31: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:22.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:21 smithi012 ceph-mon[37567]: pgmap v31: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:23.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:23 smithi116 ceph-mon[29897]: pgmap v32: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:24.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:23 smithi012 ceph-mon[91852]: pgmap v32: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:24.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:23 smithi012 ceph-mon[37567]: pgmap v32: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:25.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:25 smithi116 ceph-mon[29897]: pgmap v33: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:26.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:25 smithi012 ceph-mon[91852]: pgmap v33: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:26.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:25 smithi012 ceph-mon[37567]: pgmap v33: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:26.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:26 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:27.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:26 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:27.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:26 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:27.991 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:27 smithi012 ceph-mon[91852]: pgmap v34: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:27.991 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:27 smithi012 ceph-mon[37567]: pgmap v34: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:27.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:27 smithi116 ceph-mon[29897]: pgmap v34: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:28.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:00:27 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:00:27] "GET /metrics HTTP/1.1" 200 33774 "" "Prometheus/2.43.0" 2024-04-26T05:00:28.992 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:00:28 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:00:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi116.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:00:30.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:29 smithi012 ceph-mon[91852]: pgmap v35: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:30.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:29 smithi012 ceph-mon[37567]: pgmap v35: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:30.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:29 smithi116 ceph-mon[29897]: pgmap v35: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:32.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:31 smithi012 ceph-mon[91852]: pgmap v36: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:32.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:31 smithi012 ceph-mon[37567]: pgmap v36: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:32.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:31 smithi116 ceph-mon[29897]: pgmap v36: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:34.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:33 smithi012 ceph-mon[91852]: pgmap v37: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:34.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:33 smithi012 ceph-mon[37567]: pgmap v37: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:34.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:33 smithi116 ceph-mon[29897]: pgmap v37: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:35.403 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade resume' 2024-04-26T05:00:35.680 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:00:36.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:35 smithi012 ceph-mon[91852]: pgmap v38: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:36.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:35 smithi012 ceph-mon[37567]: pgmap v38: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:36.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:35 smithi116 ceph-mon[29897]: pgmap v38: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:37.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:36 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:00:37.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:36 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:00:37.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:36 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:00:37.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:36 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:00:37.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:36 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:00:37.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:36 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:00:37.828 INFO:teuthology.orchestra.run.smithi012.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:00:37.829 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:37 smithi012 ceph-mon[91852]: pgmap v39: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:37.829 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:37 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:00:37.829 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:37 smithi012 ceph-mon[37567]: pgmap v39: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:37.829 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:37 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:00:38.135 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:00:37 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:00:37] "GET /metrics HTTP/1.1" 200 33774 "" "Prometheus/2.43.0" 2024-04-26T05:00:38.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:37 smithi116 ceph-mon[29897]: pgmap v39: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:38.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:37 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:00:38.830 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:00:38 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:00:38.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi116.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:00:39.118 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:38 smithi012 ceph-mon[91852]: from='client.25307 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:00:39.118 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:38 smithi012 ceph-mon[91852]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:00:39.118 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:38 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:00:39.118 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:38 smithi012 ceph-mon[37567]: from='client.25307 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:00:39.118 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:38 smithi012 ceph-mon[37567]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:00:39.118 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:38 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:00:39.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:38 smithi116 ceph-mon[29897]: from='client.25307 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:00:39.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:38 smithi116 ceph-mon[29897]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:00:39.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:38 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:00:39.269 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'echo wait for majority of mons to upgrade' 2024-04-26T05:00:39.995 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:00:40.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:39 smithi012 ceph-mon[91852]: pgmap v40: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:40.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:39 smithi012 ceph-mon[37567]: pgmap v40: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:40.163 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:39 smithi116 ceph-mon[29897]: pgmap v40: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:41.513 INFO:teuthology.orchestra.run.smithi012.stdout:wait for majority of mons to upgrade 2024-04-26T05:00:41.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:41 smithi116 ceph-mon[29897]: pgmap v41: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:41.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:41 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:00:41.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:41 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:00:41.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:41 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:42.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:41 smithi012 ceph-mon[37567]: pgmap v41: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:42.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:41 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:00:42.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:41 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:00:42.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:41 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:42.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:41 smithi012 ceph-mon[91852]: pgmap v41: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:42.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:41 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:00:42.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:41 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:00:42.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:41 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:42.266 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2024-04-26T05:00:43.003 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:00:43.983 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:43 smithi012 ceph-mon[91852]: pgmap v42: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:43.983 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:43 smithi012 ceph-mon[37567]: pgmap v42: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:43.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:43 smithi116 ceph-mon[29897]: pgmap v42: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:45.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:45 smithi116 ceph-mon[29897]: pgmap v43: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:45.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:45 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:00:45.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:45 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:00:45.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:45 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:00:45.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:45 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:00:45.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:45 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:00:45.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:45 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:00:45.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:45 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:00:45.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:45 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T05:00:45.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:45 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-26T05:00:45.747 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:45 smithi012 ceph-mon[91852]: pgmap v43: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:45.747 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:45 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:00:45.747 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:45 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:00:45.747 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:45 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:00:45.747 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:45 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:00:45.748 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:45 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:00:45.748 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:45 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:00:45.748 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:45 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:00:45.748 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:45 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T05:00:45.748 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:45 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-26T05:00:45.748 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:45 smithi012 ceph-mon[37567]: pgmap v43: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:45.748 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:45 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:00:45.748 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:45 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:00:45.748 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:45 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:00:45.748 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:45 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:00:45.748 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:45 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:00:45.748 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:45 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:00:45.748 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:45 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:00:45.748 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:45 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T05:00:45.748 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:45 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-26T05:00:46.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:46 smithi116 ceph-mon[29897]: pgmap v44: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:46.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:46 smithi116 ceph-mon[29897]: pgmap v45: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:46.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:46 smithi116 ceph-mon[29897]: Upgrade: It appears safe to stop mon.c 2024-04-26T05:00:46.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:46 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/1630614795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:46.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:46 smithi012 ceph-mon[91852]: pgmap v44: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:46.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:46 smithi012 ceph-mon[91852]: pgmap v45: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:46.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:46 smithi012 ceph-mon[91852]: Upgrade: It appears safe to stop mon.c 2024-04-26T05:00:46.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:46 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/1630614795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:46.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:46 smithi012 ceph-mon[37567]: pgmap v44: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:46.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:46 smithi012 ceph-mon[37567]: pgmap v45: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:46.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:46 smithi012 ceph-mon[37567]: Upgrade: It appears safe to stop mon.c 2024-04-26T05:00:46.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:46 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/1630614795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:47.991 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:47 smithi012 ceph-mon[91852]: Upgrade: Updating mon.c 2024-04-26T05:00:47.991 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:47 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:00:47.991 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:47 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T05:00:47.992 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:47 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T05:00:47.992 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:47 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:00:47.992 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:47 smithi012 ceph-mon[91852]: Deploying daemon mon.c on smithi012 2024-04-26T05:00:47.992 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:47 smithi012 ceph-mon[37567]: Upgrade: Updating mon.c 2024-04-26T05:00:47.992 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:47 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:00:47.992 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:47 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T05:00:47.992 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:47 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T05:00:47.992 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:47 smithi012 ceph-mon[37567]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:00:47.992 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:47 smithi012 ceph-mon[37567]: Deploying daemon mon.c on smithi012 2024-04-26T05:00:48.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:47 smithi116 ceph-mon[29897]: Upgrade: Updating mon.c 2024-04-26T05:00:48.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:47 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:00:48.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:47 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T05:00:48.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:47 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T05:00:48.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:47 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:00:48.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:47 smithi116 ceph-mon[29897]: Deploying daemon mon.c on smithi012 2024-04-26T05:00:48.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:00:47 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:00:47] "GET /metrics HTTP/1.1" 200 33772 "" "Prometheus/2.43.0" 2024-04-26T05:00:48.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:48 smithi116 ceph-mon[29897]: pgmap v46: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:48.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:48 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/2974790777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:48.992 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:00:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:00:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi116.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:00:49.049 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:48 smithi012 ceph-mon[91852]: pgmap v46: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:49.049 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:48 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/2974790777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:49.049 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:48 smithi012 systemd[1]: Stopping Ceph mon.c for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T05:00:49.049 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:48 smithi012 ceph-mon[37567]: pgmap v46: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:49.049 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:48 smithi012 ceph-mon[37567]: from='client.? 172.21.15.12:0/2974790777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:00:49.340 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:49 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-c[37563]: 2024-04-26T05:00:49.047+0000 7fa505a37700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:00:49.340 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:49 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-c[37563]: 2024-04-26T05:00:49.047+0000 7fa505a37700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-04-26T05:00:49.340 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:49 smithi012 podman[95788]: 2024-04-26 05:00:49.089023503 +0000 UTC m=+0.154355618 container died 0f03d4c16528513e9afcd9c7cf63732f1f25d5bb1bcb3ff09079b69b6a66d946 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-c, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-c61472c, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-04-26T05:00:49.758 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:49 smithi012 podman[95788]: 2024-04-26 05:00:49.522774177 +0000 UTC m=+0.588106345 container cleanup 0f03d4c16528513e9afcd9c7cf63732f1f25d5bb1bcb3ff09079b69b6a66d946 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-c, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=quincy-c61472c, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240424) 2024-04-26T05:00:49.758 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:49 smithi012 bash[95788]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-c 2024-04-26T05:00:50.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:49 smithi012 podman[95803]: 2024-04-26 05:00:49.757925313 +0000 UTC m=+0.673380380 container remove 0f03d4c16528513e9afcd9c7cf63732f1f25d5bb1bcb3ff09079b69b6a66d946 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-c, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, RELEASE=quincy-c61472c, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5) 2024-04-26T05:00:50.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mon.c.service: Deactivated successfully. 2024-04-26T05:00:50.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 systemd[1]: Stopped Ceph mon.c for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T05:00:50.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mon.c.service: Consumed 7.938s CPU time. 2024-04-26T05:00:50.434 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 systemd[1]: Starting Ceph mon.c for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T05:00:50.686 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 podman[95956]: 2024-04-26 05:00:50.43400079 +0000 UTC m=+0.113974622 container create 4d75f0bd2f162ca53e9f6362694300674ca81993334806c816eb2ca3ea89a398 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-c, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-04-26T05:00:50.686 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 podman[95956]: 2024-04-26 05:00:50.341597344 +0000 UTC m=+0.021571210 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:00:50.686 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 podman[95956]: 2024-04-26 05:00:50.645849537 +0000 UTC m=+0.325823367 container init 4d75f0bd2f162ca53e9f6362694300674ca81993334806c816eb2ca3ea89a398 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-c, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T05:00:50.686 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 podman[95956]: 2024-04-26 05:00:50.649167646 +0000 UTC m=+0.329141483 container start 4d75f0bd2f162ca53e9f6362694300674ca81993334806c816eb2ca3ea89a398 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-c, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-04-26T05:00:50.686 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: set uid:gid to 167:167 (ceph:ceph) 2024-04-26T05:00:50.686 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev), process ceph-mon, pid 2 2024-04-26T05:00:50.686 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: pidfile_write: ignore empty --pid-file 2024-04-26T05:00:50.686 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: load: jerasure load: lrc 2024-04-26T05:00:50.686 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: RocksDB version: 7.9.2 2024-04-26T05:00:50.686 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Git sha 0 2024-04-26T05:00:50.686 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Compile date 2024-04-25 16:43:49 2024-04-26T05:00:50.687 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: DB SUMMARY 2024-04-26T05:00:50.687 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: DB Session ID: UV98YE1R5KGZVMLQUSBC 2024-04-26T05:00:50.687 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: CURRENT file: CURRENT 2024-04-26T05:00:50.687 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: IDENTITY file: IDENTITY 2024-04-26T05:00:50.687 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1084 Bytes 2024-04-26T05:00:50.687 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2024-04-26T05:00:50.687 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 1030925 ; 2024-04-26T05:00:50.687 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.error_if_exists: 0 2024-04-26T05:00:50.689 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.create_if_missing: 0 2024-04-26T05:00:50.689 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.paranoid_checks: 1 2024-04-26T05:00:50.689 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-26T05:00:50.689 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-26T05:00:50.689 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-26T05:00:50.689 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.env: 0x55eba0a9bc60 2024-04-26T05:00:50.690 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.fs: PosixFileSystem 2024-04-26T05:00:50.690 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.info_log: 0x55eba29a1aa0 2024-04-26T05:00:50.690 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.max_file_opening_threads: 16 2024-04-26T05:00:50.690 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.statistics: (nil) 2024-04-26T05:00:50.690 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.use_fsync: 0 2024-04-26T05:00:50.690 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.max_log_file_size: 0 2024-04-26T05:00:50.690 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-26T05:00:50.690 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-26T05:00:50.690 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.keep_log_file_num: 1000 2024-04-26T05:00:50.690 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.recycle_log_file_num: 0 2024-04-26T05:00:50.690 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.allow_fallocate: 1 2024-04-26T05:00:50.691 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.allow_mmap_reads: 0 2024-04-26T05:00:50.691 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.allow_mmap_writes: 0 2024-04-26T05:00:50.691 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.use_direct_reads: 0 2024-04-26T05:00:50.691 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T05:00:50.691 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T05:00:50.691 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.db_log_dir: 2024-04-26T05:00:50.691 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.wal_dir: 2024-04-26T05:00:50.691 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T05:00:50.691 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T05:00:50.691 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T05:00:50.691 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T05:00:50.691 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T05:00:50.691 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T05:00:50.691 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T05:00:50.691 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T05:00:50.692 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.write_buffer_manager: 0x55eba2974820 2024-04-26T05:00:50.692 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T05:00:50.692 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T05:00:50.692 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T05:00:50.692 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T05:00:50.692 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T05:00:50.692 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T05:00:50.692 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T05:00:50.692 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T05:00:50.692 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.unordered_write: 0 2024-04-26T05:00:50.693 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T05:00:50.693 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T05:00:50.693 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T05:00:50.693 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T05:00:50.693 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.row_cache: None 2024-04-26T05:00:50.693 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.wal_filter: None 2024-04-26T05:00:50.693 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T05:00:50.693 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T05:00:50.693 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.two_write_queues: 0 2024-04-26T05:00:50.693 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T05:00:50.693 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.wal_compression: 0 2024-04-26T05:00:50.693 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.atomic_flush: 0 2024-04-26T05:00:50.693 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T05:00:50.693 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T05:00:50.693 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T05:00:50.694 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.log_readahead_size: 0 2024-04-26T05:00:50.694 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T05:00:50.694 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T05:00:50.694 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T05:00:50.694 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T05:00:50.694 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T05:00:50.694 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T05:00:50.694 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-26T05:00:50.694 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.max_background_jobs: 2 2024-04-26T05:00:50.694 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.max_background_compactions: -1 2024-04-26T05:00:50.694 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.max_subcompactions: 1 2024-04-26T05:00:50.694 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T05:00:50.694 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T05:00:50.695 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T05:00:50.695 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T05:00:50.695 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T05:00:50.695 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T05:00:50.695 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T05:00:50.695 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T05:00:50.695 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.max_open_files: -1 2024-04-26T05:00:50.695 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T05:00:50.695 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T05:00:50.695 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T05:00:50.695 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T05:00:50.695 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.max_background_flushes: -1 2024-04-26T05:00:50.695 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Compression algorithms supported: 2024-04-26T05:00:50.695 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T05:00:50.696 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: kZSTD supported: 0 2024-04-26T05:00:50.696 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: kXpressCompression supported: 0 2024-04-26T05:00:50.696 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T05:00:50.696 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: kZlibCompression supported: 1 2024-04-26T05:00:50.696 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: kSnappyCompression supported: 1 2024-04-26T05:00:50.696 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: kLZ4Compression supported: 1 2024-04-26T05:00:50.696 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: kBZip2Compression supported: 0 2024-04-26T05:00:50.696 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T05:00:50.696 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-26T05:00:50.696 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-04-26T05:00:50.696 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-26T05:00:50.696 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T05:00:50.696 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.merge_operator: 2024-04-26T05:00:50.696 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.compaction_filter: None 2024-04-26T05:00:50.697 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T05:00:50.697 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T05:00:50.697 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T05:00:50.697 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T05:00:50.697 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55eba29a1ba0) 2024-04-26T05:00:50.697 INFO:journalctl@ceph.mon.c.smithi012.stdout: cache_index_and_filter_blocks: 1 2024-04-26T05:00:50.697 INFO:journalctl@ceph.mon.c.smithi012.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T05:00:50.697 INFO:journalctl@ceph.mon.c.smithi012.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T05:00:50.697 INFO:journalctl@ceph.mon.c.smithi012.stdout: pin_top_level_index_and_filter: 1 2024-04-26T05:00:50.697 INFO:journalctl@ceph.mon.c.smithi012.stdout: index_type: 0 2024-04-26T05:00:50.697 INFO:journalctl@ceph.mon.c.smithi012.stdout: data_block_index_type: 0 2024-04-26T05:00:50.697 INFO:journalctl@ceph.mon.c.smithi012.stdout: index_shortening: 1 2024-04-26T05:00:50.697 INFO:journalctl@ceph.mon.c.smithi012.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T05:00:50.697 INFO:journalctl@ceph.mon.c.smithi012.stdout: checksum: 4 2024-04-26T05:00:50.697 INFO:journalctl@ceph.mon.c.smithi012.stdout: no_block_cache: 0 2024-04-26T05:00:50.698 INFO:journalctl@ceph.mon.c.smithi012.stdout: block_cache: 0x55eba29a5090 2024-04-26T05:00:50.698 INFO:journalctl@ceph.mon.c.smithi012.stdout: block_cache_name: BinnedLRUCache 2024-04-26T05:00:50.698 INFO:journalctl@ceph.mon.c.smithi012.stdout: block_cache_options: 2024-04-26T05:00:50.698 INFO:journalctl@ceph.mon.c.smithi012.stdout: capacity : 536870912 2024-04-26T05:00:50.698 INFO:journalctl@ceph.mon.c.smithi012.stdout: num_shard_bits : 4 2024-04-26T05:00:50.698 INFO:journalctl@ceph.mon.c.smithi012.stdout: strict_capacity_limit : 0 2024-04-26T05:00:50.698 INFO:journalctl@ceph.mon.c.smithi012.stdout: high_pri_pool_ratio: 0.000 2024-04-26T05:00:50.698 INFO:journalctl@ceph.mon.c.smithi012.stdout: block_cache_compressed: (nil) 2024-04-26T05:00:50.698 INFO:journalctl@ceph.mon.c.smithi012.stdout: persistent_cache: (nil) 2024-04-26T05:00:50.698 INFO:journalctl@ceph.mon.c.smithi012.stdout: block_size: 4096 2024-04-26T05:00:50.698 INFO:journalctl@ceph.mon.c.smithi012.stdout: block_size_deviation: 10 2024-04-26T05:00:50.698 INFO:journalctl@ceph.mon.c.smithi012.stdout: block_restart_interval: 16 2024-04-26T05:00:50.699 INFO:journalctl@ceph.mon.c.smithi012.stdout: index_block_restart_interval: 1 2024-04-26T05:00:50.699 INFO:journalctl@ceph.mon.c.smithi012.stdout: metadata_block_size: 4096 2024-04-26T05:00:50.699 INFO:journalctl@ceph.mon.c.smithi012.stdout: partition_filters: 0 2024-04-26T05:00:50.699 INFO:journalctl@ceph.mon.c.smithi012.stdout: use_delta_encoding: 1 2024-04-26T05:00:50.699 INFO:journalctl@ceph.mon.c.smithi012.stdout: filter_policy: bloomfilter 2024-04-26T05:00:50.699 INFO:journalctl@ceph.mon.c.smithi012.stdout: whole_key_filtering: 1 2024-04-26T05:00:50.699 INFO:journalctl@ceph.mon.c.smithi012.stdout: verify_compression: 0 2024-04-26T05:00:50.699 INFO:journalctl@ceph.mon.c.smithi012.stdout: read_amp_bytes_per_bit: 0 2024-04-26T05:00:50.699 INFO:journalctl@ceph.mon.c.smithi012.stdout: format_version: 5 2024-04-26T05:00:50.699 INFO:journalctl@ceph.mon.c.smithi012.stdout: enable_index_compression: 1 2024-04-26T05:00:50.699 INFO:journalctl@ceph.mon.c.smithi012.stdout: block_align: 0 2024-04-26T05:00:50.699 INFO:journalctl@ceph.mon.c.smithi012.stdout: max_auto_readahead_size: 262144 2024-04-26T05:00:50.699 INFO:journalctl@ceph.mon.c.smithi012.stdout: prepopulate_block_cache: 0 2024-04-26T05:00:50.699 INFO:journalctl@ceph.mon.c.smithi012.stdout: initial_auto_readahead_size: 8192 2024-04-26T05:00:50.699 INFO:journalctl@ceph.mon.c.smithi012.stdout: num_file_reads_for_auto_readahead: 2 2024-04-26T05:00:50.699 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T05:00:50.700 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T05:00:50.700 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.compression: NoCompression 2024-04-26T05:00:50.700 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T05:00:50.700 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T05:00:50.700 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T05:00:50.700 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.num_levels: 7 2024-04-26T05:00:50.700 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T05:00:50.700 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T05:00:50.700 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T05:00:50.700 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T05:00:50.700 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T05:00:50.700 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T05:00:50.700 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T05:00:50.700 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T05:00:50.700 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T05:00:50.701 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T05:00:50.701 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-26T05:00:50.701 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-26T05:00:50.701 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T05:00:50.701 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T05:00:50.701 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T05:00:50.701 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T05:00:50.701 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T05:00:50.701 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-26T05:00:50.701 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T05:00:50.701 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T05:00:50.701 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-26T05:00:50.702 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T05:00:50.702 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T05:00:50.702 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T05:00:50.702 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T05:00:50.702 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T05:00:50.702 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T05:00:50.702 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T05:00:50.702 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T05:00:50.702 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T05:00:50.702 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T05:00:50.702 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T05:00:50.702 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T05:00:50.703 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T05:00:50.703 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T05:00:50.703 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T05:00:50.703 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T05:00:50.703 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T05:00:50.703 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-26T05:00:50.703 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.arena_block_size: 1048576 2024-04-26T05:00:50.704 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T05:00:50.704 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T05:00:50.704 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T05:00:50.704 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T05:00:50.705 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T05:00:50.705 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T05:00:50.705 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T05:00:50.705 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T05:00:50.705 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T05:00:50.705 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T05:00:50.705 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T05:00:50.705 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T05:00:50.705 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T05:00:50.705 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.table_properties_collectors: 2024-04-26T05:00:50.705 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.inplace_update_support: 0 2024-04-26T05:00:50.705 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T05:00:50.705 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T05:00:50.705 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T05:00:50.706 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T05:00:50.706 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.bloom_locality: 0 2024-04-26T05:00:50.706 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.max_successive_merges: 0 2024-04-26T05:00:50.706 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T05:00:50.706 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T05:00:50.706 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T05:00:50.706 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T05:00:50.706 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.ttl: 2592000 2024-04-26T05:00:50.706 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T05:00:50.706 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-26T05:00:50.706 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-26T05:00:50.706 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.enable_blob_files: false 2024-04-26T05:00:50.706 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.min_blob_size: 0 2024-04-26T05:00:50.706 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T05:00:50.706 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T05:00:50.707 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T05:00:50.707 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T05:00:50.707 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-26T05:00:50.707 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-26T05:00:50.707 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.blob_file_starting_level: 0 2024-04-26T05:00:50.707 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-26T05:00:50.707 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-04-26T05:00:50.707 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T05:00:50.707 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 12967, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T05:00:50.707 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-04-26T05:00:50.707 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 5f895fa6-cde5-4de8-91cc-33f05743dd6b 2024-04-26T05:00:50.707 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714107650678758, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-04-26T05:00:50.707 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-04-26T05:00:50.707 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714107650682435, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 660433, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12972, "largest_seqno": 13757, "table_properties": {"data_size": 656458, "index_size": 2193, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 773, "raw_key_size": 6540, "raw_average_key_size": 22, "raw_value_size": 649854, "raw_average_value_size": 2202, "num_data_blocks": 98, "num_entries": 295, "num_filter_entries": 295, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1714107650, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "5f895fa6-cde5-4de8-91cc-33f05743dd6b", "db_session_id": "UV98YE1R5KGZVMLQUSBC", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-04-26T05:00:50.708 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714107650682527, "job": 1, "event": "recovery_finished"} 2024-04-26T05:00:50.708 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-04-26T05:00:50.708 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T05:00:50.708 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T05:00:50.708 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55eba2a96000 2024-04-26T05:00:50.708 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: DB pointer 0x55eba2a80000 2024-04-26T05:00:50.708 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-26T05:00:50.708 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-26T05:00:50.708 INFO:journalctl@ceph.mon.c.smithi012.stdout: ** DB Stats ** 2024-04-26T05:00:50.708 INFO:journalctl@ceph.mon.c.smithi012.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T05:00:50.708 INFO:journalctl@ceph.mon.c.smithi012.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T05:00:50.708 INFO:journalctl@ceph.mon.c.smithi012.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T05:00:50.708 INFO:journalctl@ceph.mon.c.smithi012.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T05:00:50.708 INFO:journalctl@ceph.mon.c.smithi012.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T05:00:50.709 INFO:journalctl@ceph.mon.c.smithi012.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T05:00:50.709 INFO:journalctl@ceph.mon.c.smithi012.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T05:00:50.709 INFO:journalctl@ceph.mon.c.smithi012.stdout: 2024-04-26T05:00:50.709 INFO:journalctl@ceph.mon.c.smithi012.stdout: ** Compaction Stats [default] ** 2024-04-26T05:00:50.709 INFO:journalctl@ceph.mon.c.smithi012.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T05:00:50.709 INFO:journalctl@ceph.mon.c.smithi012.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-26T05:00:50.709 INFO:journalctl@ceph.mon.c.smithi012.stdout: L0 1/0 644.95 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 319.7 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-04-26T05:00:50.709 INFO:journalctl@ceph.mon.c.smithi012.stdout: L6 1/0 8.50 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-26T05:00:50.709 INFO:journalctl@ceph.mon.c.smithi012.stdout: Sum 2/0 9.13 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 319.7 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-04-26T05:00:50.709 INFO:journalctl@ceph.mon.c.smithi012.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 319.7 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-04-26T05:00:50.709 INFO:journalctl@ceph.mon.c.smithi012.stdout: 2024-04-26T05:00:50.709 INFO:journalctl@ceph.mon.c.smithi012.stdout: ** Compaction Stats [default] ** 2024-04-26T05:00:50.709 INFO:journalctl@ceph.mon.c.smithi012.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T05:00:50.709 INFO:journalctl@ceph.mon.c.smithi012.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T05:00:50.709 INFO:journalctl@ceph.mon.c.smithi012.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 319.7 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-04-26T05:00:50.709 INFO:journalctl@ceph.mon.c.smithi012.stdout: 2024-04-26T05:00:50.710 INFO:journalctl@ceph.mon.c.smithi012.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-26T05:00:50.710 INFO:journalctl@ceph.mon.c.smithi012.stdout: 2024-04-26T05:00:50.710 INFO:journalctl@ceph.mon.c.smithi012.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T05:00:50.710 INFO:journalctl@ceph.mon.c.smithi012.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-04-26T05:00:50.710 INFO:journalctl@ceph.mon.c.smithi012.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T05:00:50.710 INFO:journalctl@ceph.mon.c.smithi012.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T05:00:50.710 INFO:journalctl@ceph.mon.c.smithi012.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T05:00:50.710 INFO:journalctl@ceph.mon.c.smithi012.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T05:00:50.710 INFO:journalctl@ceph.mon.c.smithi012.stdout: Cumulative compaction: 0.00 GB write, 108.37 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T05:00:50.710 INFO:journalctl@ceph.mon.c.smithi012.stdout: Interval compaction: 0.00 GB write, 108.37 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T05:00:50.710 INFO:journalctl@ceph.mon.c.smithi012.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T05:00:50.710 INFO:journalctl@ceph.mon.c.smithi012.stdout: Block cache BinnedLRUCache@0x55eba29a5090#2 capacity: 512.00 MB usage: 3.23 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-04-26T05:00:50.710 INFO:journalctl@ceph.mon.c.smithi012.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.91 KB,0.000172853%) IndexBlock(1,2.33 KB,0.000444055%) Misc(1,0.00 KB,0%) 2024-04-26T05:00:50.711 INFO:journalctl@ceph.mon.c.smithi012.stdout: 2024-04-26T05:00:50.711 INFO:journalctl@ceph.mon.c.smithi012.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T05:00:50.711 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: starting mon.c rank 2 at public addrs [v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0] at bind addrs [v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T05:00:50.711 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: mon.c@-1(???) e3 preinit fsid 0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T05:00:50.711 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: mon.c@-1(???).mds e9 new map 2024-04-26T05:00:50.711 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: mon.c@-1(???).mds e9 print_map 2024-04-26T05:00:50.711 INFO:journalctl@ceph.mon.c.smithi012.stdout: e9 2024-04-26T05:00:50.711 INFO:journalctl@ceph.mon.c.smithi012.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T05:00:50.711 INFO:journalctl@ceph.mon.c.smithi012.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T05:00:50.711 INFO:journalctl@ceph.mon.c.smithi012.stdout: legacy client fscid: 1 2024-04-26T05:00:50.711 INFO:journalctl@ceph.mon.c.smithi012.stdout: 2024-04-26T05:00:50.711 INFO:journalctl@ceph.mon.c.smithi012.stdout: Filesystem 'foo' (1) 2024-04-26T05:00:50.711 INFO:journalctl@ceph.mon.c.smithi012.stdout: fs_name foo 2024-04-26T05:00:50.711 INFO:journalctl@ceph.mon.c.smithi012.stdout: epoch 8 2024-04-26T05:00:50.712 INFO:journalctl@ceph.mon.c.smithi012.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-26T05:00:50.712 INFO:journalctl@ceph.mon.c.smithi012.stdout: created 2024-04-26T04:55:07.451848+0000 2024-04-26T05:00:50.712 INFO:journalctl@ceph.mon.c.smithi012.stdout: modified 2024-04-26T04:55:21.872954+0000 2024-04-26T05:00:50.712 INFO:journalctl@ceph.mon.c.smithi012.stdout: tableserver 0 2024-04-26T05:00:50.712 INFO:journalctl@ceph.mon.c.smithi012.stdout: root 0 2024-04-26T05:00:50.712 INFO:journalctl@ceph.mon.c.smithi012.stdout: session_timeout 60 2024-04-26T05:00:50.712 INFO:journalctl@ceph.mon.c.smithi012.stdout: session_autoclose 300 2024-04-26T05:00:50.712 INFO:journalctl@ceph.mon.c.smithi012.stdout: max_file_size 1099511627776 2024-04-26T05:00:50.712 INFO:journalctl@ceph.mon.c.smithi012.stdout: max_xattr_size 65536 2024-04-26T05:00:50.712 INFO:journalctl@ceph.mon.c.smithi012.stdout: required_client_features {} 2024-04-26T05:00:50.712 INFO:journalctl@ceph.mon.c.smithi012.stdout: last_failure 0 2024-04-26T05:00:50.712 INFO:journalctl@ceph.mon.c.smithi012.stdout: last_failure_osd_epoch 0 2024-04-26T05:00:50.712 INFO:journalctl@ceph.mon.c.smithi012.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T05:00:50.712 INFO:journalctl@ceph.mon.c.smithi012.stdout: max_mds 1 2024-04-26T05:00:50.713 INFO:journalctl@ceph.mon.c.smithi012.stdout: in 0 2024-04-26T05:00:50.713 INFO:journalctl@ceph.mon.c.smithi012.stdout: up {0=14664} 2024-04-26T05:00:50.713 INFO:journalctl@ceph.mon.c.smithi012.stdout: failed 2024-04-26T05:00:50.713 INFO:journalctl@ceph.mon.c.smithi012.stdout: damaged 2024-04-26T05:00:50.713 INFO:journalctl@ceph.mon.c.smithi012.stdout: stopped 2024-04-26T05:00:50.713 INFO:journalctl@ceph.mon.c.smithi012.stdout: data_pools [4] 2024-04-26T05:00:50.713 INFO:journalctl@ceph.mon.c.smithi012.stdout: metadata_pool 3 2024-04-26T05:00:50.713 INFO:journalctl@ceph.mon.c.smithi012.stdout: inline_data disabled 2024-04-26T05:00:50.713 INFO:journalctl@ceph.mon.c.smithi012.stdout: balancer 2024-04-26T05:00:50.713 INFO:journalctl@ceph.mon.c.smithi012.stdout: bal_rank_mask -1 2024-04-26T05:00:50.713 INFO:journalctl@ceph.mon.c.smithi012.stdout: standby_count_wanted 1 2024-04-26T05:00:50.713 INFO:journalctl@ceph.mon.c.smithi012.stdout: qdb_cluster leader: 0 members: 2024-04-26T05:00:50.713 INFO:journalctl@ceph.mon.c.smithi012.stdout: [mds.foo.smithi012.fdqwmo{0:14664} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.12:6834/963323745,v1:172.21.15.12:6835/963323745] compat {c=[1],r=[1],i=[7ff]}] 2024-04-26T05:00:50.713 INFO:journalctl@ceph.mon.c.smithi012.stdout: 2024-04-26T05:00:50.714 INFO:journalctl@ceph.mon.c.smithi012.stdout: 2024-04-26T05:00:50.714 INFO:journalctl@ceph.mon.c.smithi012.stdout: Standby daemons: 2024-04-26T05:00:50.714 INFO:journalctl@ceph.mon.c.smithi012.stdout: 2024-04-26T05:00:50.714 INFO:journalctl@ceph.mon.c.smithi012.stdout: [mds.foo.smithi116.kduyef{-1:24520} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.116:6832/529792701,v1:172.21.15.116:6833/529792701] compat {c=[1],r=[1],i=[7ff]}] 2024-04-26T05:00:50.714 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: mon.c@-1(???).osd e80 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-26T05:00:50.714 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: mon.c@-1(???).osd e80 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T05:00:50.714 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: mon.c@-1(???).osd e80 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T05:00:50.714 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: mon.c@-1(???).osd e80 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T05:00:50.714 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 ceph-mon[95978]: mon.c@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-04-26T05:00:51.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 bash[95956]: 4d75f0bd2f162ca53e9f6362694300674ca81993334806c816eb2ca3ea89a398 2024-04-26T05:00:51.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:50 smithi012 systemd[1]: Started Ceph mon.c for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T05:00:57.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:57 smithi116 ceph-mon[29897]: mon.c calling monitor election 2024-04-26T05:00:57.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:57 smithi116 ceph-mon[29897]: mon.b calling monitor election 2024-04-26T05:00:57.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:57 smithi116 ceph-mon[29897]: mon.a calling monitor election 2024-04-26T05:00:57.494 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:57 smithi116 ceph-mon[29897]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T05:00:57.494 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:57 smithi116 ceph-mon[29897]: monmap epoch 3 2024-04-26T05:00:57.494 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:57 smithi116 ceph-mon[29897]: fsid 0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T05:00:57.494 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:57 smithi116 ceph-mon[29897]: last_changed 2024-04-26T04:49:56.953431+0000 2024-04-26T05:00:57.494 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:57 smithi116 ceph-mon[29897]: created 2024-04-26T04:48:15.190556+0000 2024-04-26T05:00:57.494 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:57 smithi116 ceph-mon[29897]: min_mon_release 17 (quincy) 2024-04-26T05:00:57.494 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:57 smithi116 ceph-mon[29897]: election_strategy: 1 2024-04-26T05:00:57.494 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:57 smithi116 ceph-mon[29897]: 0: [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] mon.a 2024-04-26T05:00:57.494 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:57 smithi116 ceph-mon[29897]: 1: [v2:172.21.15.116:3300/0,v1:172.21.15.116:6789/0] mon.b 2024-04-26T05:00:57.494 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:57 smithi116 ceph-mon[29897]: 2: [v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0] mon.c 2024-04-26T05:00:57.494 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:57 smithi116 ceph-mon[29897]: fsmap foo:1 {0=foo.smithi012.fdqwmo=up:active} 1 up:standby 2024-04-26T05:00:57.494 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:57 smithi116 ceph-mon[29897]: osdmap e80: 8 total, 8 up, 8 in 2024-04-26T05:00:57.494 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:57 smithi116 ceph-mon[29897]: mgrmap e38: y(active, since 90s), standbys: x 2024-04-26T05:00:57.495 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:57 smithi116 ceph-mon[29897]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-04-26T05:00:57.495 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:57 smithi116 ceph-mon[29897]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T05:00:57.495 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:57 smithi116 ceph-mon[29897]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T05:00:57.495 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:57 smithi116 ceph-mon[29897]: daemon grafana.a on smithi116 is in error state 2024-04-26T05:00:57.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[91852]: mon.c calling monitor election 2024-04-26T05:00:57.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[91852]: mon.b calling monitor election 2024-04-26T05:00:57.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[91852]: mon.a calling monitor election 2024-04-26T05:00:57.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[91852]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T05:00:57.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[91852]: monmap epoch 3 2024-04-26T05:00:57.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[91852]: fsid 0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T05:00:57.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[91852]: last_changed 2024-04-26T04:49:56.953431+0000 2024-04-26T05:00:57.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[91852]: created 2024-04-26T04:48:15.190556+0000 2024-04-26T05:00:57.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[91852]: min_mon_release 17 (quincy) 2024-04-26T05:00:57.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[91852]: election_strategy: 1 2024-04-26T05:00:57.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[91852]: 0: [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] mon.a 2024-04-26T05:00:57.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[91852]: 1: [v2:172.21.15.116:3300/0,v1:172.21.15.116:6789/0] mon.b 2024-04-26T05:00:57.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[91852]: 2: [v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0] mon.c 2024-04-26T05:00:57.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[91852]: fsmap foo:1 {0=foo.smithi012.fdqwmo=up:active} 1 up:standby 2024-04-26T05:00:57.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[91852]: osdmap e80: 8 total, 8 up, 8 in 2024-04-26T05:00:57.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[91852]: mgrmap e38: y(active, since 90s), standbys: x 2024-04-26T05:00:57.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[91852]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-04-26T05:00:57.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[91852]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T05:00:57.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[91852]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T05:00:57.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[91852]: daemon grafana.a on smithi116 is in error state 2024-04-26T05:00:57.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[95978]: mon.c calling monitor election 2024-04-26T05:00:57.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[95978]: mon.b calling monitor election 2024-04-26T05:00:57.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[95978]: mon.a calling monitor election 2024-04-26T05:00:57.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[95978]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T05:00:57.516 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[95978]: monmap epoch 3 2024-04-26T05:00:57.516 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[95978]: fsid 0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T05:00:57.516 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[95978]: last_changed 2024-04-26T04:49:56.953431+0000 2024-04-26T05:00:57.516 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[95978]: created 2024-04-26T04:48:15.190556+0000 2024-04-26T05:00:57.516 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[95978]: min_mon_release 17 (quincy) 2024-04-26T05:00:57.516 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[95978]: election_strategy: 1 2024-04-26T05:00:57.516 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[95978]: 0: [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] mon.a 2024-04-26T05:00:57.516 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[95978]: 1: [v2:172.21.15.116:3300/0,v1:172.21.15.116:6789/0] mon.b 2024-04-26T05:00:57.516 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[95978]: 2: [v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0] mon.c 2024-04-26T05:00:57.516 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[95978]: fsmap foo:1 {0=foo.smithi012.fdqwmo=up:active} 1 up:standby 2024-04-26T05:00:57.516 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[95978]: osdmap e80: 8 total, 8 up, 8 in 2024-04-26T05:00:57.516 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[95978]: mgrmap e38: y(active, since 90s), standbys: x 2024-04-26T05:00:57.516 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[95978]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-04-26T05:00:57.516 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[95978]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T05:00:57.517 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[95978]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T05:00:57.517 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-mon[95978]: daemon grafana.a on smithi116 is in error state 2024-04-26T05:00:58.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:00:57 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:00:57] "GET /metrics HTTP/1.1" 200 33836 "" "Prometheus/2.43.0" 2024-04-26T05:00:58.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:58 smithi116 ceph-mon[29897]: pgmap v51: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:58.992 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:00:58 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:00:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi116.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:00:59.027 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-26T05:00:59.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:58 smithi012 ceph-mon[91852]: pgmap v51: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:59.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:58 smithi012 ceph-mon[95978]: pgmap v51: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:00.229 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:59 smithi012 ceph-mon[95978]: mgrmap e39: y(active, since 92s), standbys: x 2024-04-26T05:01:00.230 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:59 smithi012 ceph-mon[95978]: pgmap v52: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T05:01:00.230 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:59 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/1723412483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:01:00.230 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:59 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:00.230 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:00:59 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:00.230 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:59 smithi012 ceph-mon[91852]: mgrmap e39: y(active, since 92s), standbys: x 2024-04-26T05:01:00.230 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:59 smithi012 ceph-mon[91852]: pgmap v52: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T05:01:00.230 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:59 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/1723412483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:01:00.230 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:59 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:00.230 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:00:59 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:00.241 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:01:00 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:01:00.088Z caller=alerting.go:366 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephMonDown: template: __alert_CephMonDown:1:427: executing \"__alert_CephMonDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"mon.a\"} on the right hand-side of the operation: [{ceph_daemon=\"mon.a\", ceph_version=\"ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", public_addr=\"172.21.15.12\", rank=\"0\"}, {ceph_daemon=\"mon.a\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", hostname=\"smithi012\", instance=\"smithi116.front.sepia.ceph.com:9283\", job=\"ceph\", public_addr=\"172.21.15.12\", rank=\"0\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-04-26T05:01:00.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:59 smithi116 ceph-mon[29897]: mgrmap e39: y(active, since 92s), standbys: x 2024-04-26T05:01:00.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:59 smithi116 ceph-mon[29897]: pgmap v52: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T05:01:00.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:59 smithi116 ceph-mon[29897]: from='client.? 172.21.15.12:0/1723412483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:01:00.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:59 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:00.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:00:59 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:00.255 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-26T05:01:00.526 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:01:00.967 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:01:00 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T05:01:00.691+0000 7f0e1e4f1700 -1 mgr.server handle_report got status from non-daemon mon.c 2024-04-26T05:01:02.736 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T05:01:02.737 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.a smithi012 *:9093,9094 running (3m) 2s ago 6m 24.7M - 0.25.0 c8568f914cd2 217bb19e153b 2024-04-26T05:01:02.737 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.a smithi116 *:3000 error 22s ago 6m - - 2024-04-26T05:01:02.737 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foo.smithi012.fdqwmo smithi012 running (5m) 2s ago 5m 16.5M - 17.2.7-902-gc61472c8 0de407c1b3af 53f6c82f01b4 2024-04-26T05:01:02.737 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foo.smithi116.kduyef smithi116 running (5m) 22s ago 5m 15.3M - 17.2.7-902-gc61472c8 0de407c1b3af 4c0a0b911593 2024-04-26T05:01:02.737 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.x smithi116 *:8443,9283,8765 running (2m) 22s ago 10m 450M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 5499bf989999 2024-04-26T05:01:02.737 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.y smithi012 *:8443,9283,8765 running (2m) 2s ago 12m 500M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 3a46d0031772 2024-04-26T05:01:02.737 INFO:teuthology.orchestra.run.smithi012.stdout:mon.a smithi012 running (105s) 2s ago 12m 40.9M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 db1f873383b0 2024-04-26T05:01:02.737 INFO:teuthology.orchestra.run.smithi012.stdout:mon.b smithi116 running (11m) 22s ago 11m 47.2M 2048M 17.2.7-902-gc61472c8 0de407c1b3af e6cd16f01417 2024-04-26T05:01:02.737 INFO:teuthology.orchestra.run.smithi012.stdout:mon.c smithi012 running (12s) 2s ago 11m 17.2M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 4d75f0bd2f16 2024-04-26T05:01:02.737 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.a smithi012 *:9100 running (2m) 2s ago 6m 16.0M - 1.5.0 0da6a335fe13 6390338999c1 2024-04-26T05:01:02.737 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.b smithi116 *:9100 running (2m) 22s ago 6m 16.5M - 1.5.0 0da6a335fe13 7d0b1e809531 2024-04-26T05:01:02.737 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (10m) 2s ago 10m 47.7M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 25b6843697db 2024-04-26T05:01:02.738 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (10m) 2s ago 10m 47.0M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 8117a7da3ae3 2024-04-26T05:01:02.738 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (9m) 2s ago 9m 53.1M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 432af63060b8 2024-04-26T05:01:02.738 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (9m) 2s ago 9m 47.0M 2481M 17.2.7-902-gc61472c8 0de407c1b3af f65d4289a970 2024-04-26T05:01:02.738 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi116 running (8m) 22s ago 8m 46.4M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 6b0bbf77cb39 2024-04-26T05:01:02.738 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi116 running (8m) 22s ago 8m 51.4M 2480M 17.2.7-902-gc61472c8 0de407c1b3af a551ef15fa7e 2024-04-26T05:01:02.738 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi116 running (8m) 22s ago 8m 51.0M 2480M 17.2.7-902-gc61472c8 0de407c1b3af cdc0eb6f4fa2 2024-04-26T05:01:02.738 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi116 running (7m) 22s ago 7m 50.9M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 3f6689213fda 2024-04-26T05:01:02.738 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.a smithi116 *:9095 running (2m) 22s ago 7m 57.5M - 2.43.0 a07b618ecd1d e9e5b93523ae 2024-04-26T05:01:02.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:02 smithi116 ceph-mon[29897]: pgmap v53: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T05:01:02.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:02 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:02.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:02 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:03.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:02 smithi012 ceph-mon[91852]: pgmap v53: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T05:01:03.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:02 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:03.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:02 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:03.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:02 smithi012 ceph-mon[95978]: pgmap v53: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T05:01:03.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:02 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:03.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:02 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:03.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:03 smithi012 ceph-mon[91852]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:01:03.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:03 smithi012 ceph-mon[95978]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:01:03.882 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade pause' 2024-04-26T05:01:03.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:03 smithi116 ceph-mon[29897]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:01:04.287 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:01:04.951 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:04 smithi012 ceph-mon[91852]: pgmap v54: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T05:01:04.951 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:04 smithi012 ceph-mon[95978]: pgmap v54: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T05:01:04.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:04 smithi116 ceph-mon[29897]: pgmap v54: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T05:01:06.834 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:06 smithi012 ceph-mon[91852]: pgmap v55: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T05:01:06.835 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:06 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:06.835 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:06 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:06.835 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:06 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:06.835 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:06 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:01:06.835 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:06 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:06.835 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:06 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:01:06.835 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:06 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:01:06.835 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:06 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T05:01:06.835 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:06 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-26T05:01:06.835 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:06 smithi012 ceph-mon[95978]: pgmap v55: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T05:01:06.836 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:06 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:06.836 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:06 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:06.836 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:06 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:06.836 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:06 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:01:06.836 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:06 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:06.836 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:06 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:01:06.836 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:06 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:01:06.836 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:06 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T05:01:06.836 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:06 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-26T05:01:06.966 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:06 smithi116 ceph-mon[29897]: pgmap v55: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-26T05:01:06.966 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:06 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:06.966 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:06 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:06.966 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:06 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:06.967 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:06 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:01:06.967 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:06 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:06.967 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:06 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:01:06.967 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:06 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:01:06.967 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:06 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T05:01:06.967 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:06 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-26T05:01:07.289 INFO:teuthology.orchestra.run.smithi012.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:01:07.987 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:07 smithi012 ceph-mon[91852]: pgmap v56: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 276 B/s wr, 0 op/s 2024-04-26T05:01:07.987 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:07 smithi012 ceph-mon[91852]: pgmap v57: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 336 B/s wr, 0 op/s 2024-04-26T05:01:07.987 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:07 smithi012 ceph-mon[91852]: Upgrade: It appears safe to stop mon.b 2024-04-26T05:01:07.987 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:07 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:07.987 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:07 smithi012 ceph-mon[95978]: pgmap v56: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 276 B/s wr, 0 op/s 2024-04-26T05:01:07.987 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:07 smithi012 ceph-mon[95978]: pgmap v57: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 336 B/s wr, 0 op/s 2024-04-26T05:01:07.987 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:07 smithi012 ceph-mon[95978]: Upgrade: It appears safe to stop mon.b 2024-04-26T05:01:07.987 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:07 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:07.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:07 smithi116 ceph-mon[29897]: pgmap v56: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 276 B/s wr, 0 op/s 2024-04-26T05:01:07.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:07 smithi116 ceph-mon[29897]: pgmap v57: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 336 B/s wr, 0 op/s 2024-04-26T05:01:07.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:07 smithi116 ceph-mon[29897]: Upgrade: It appears safe to stop mon.b 2024-04-26T05:01:07.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:07 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:08.015 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'sleep 60' 2024-04-26T05:01:08.262 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:01:07 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:01:07] "GET /metrics HTTP/1.1" 200 33836 "" "Prometheus/2.43.0" 2024-04-26T05:01:08.285 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:01:08.826 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:01:08 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:01:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi116.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:01:08.826 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:08 smithi116 ceph-mon[29897]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:01:08.826 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:08 smithi116 ceph-mon[29897]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:01:08.826 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:08 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:08.827 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:08 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T05:01:08.827 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:08 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T05:01:08.827 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:08 smithi116 ceph-mon[29897]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:09.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:08 smithi012 ceph-mon[91852]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:01:09.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:08 smithi012 ceph-mon[91852]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:01:09.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:08 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:09.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:08 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T05:01:09.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:08 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T05:01:09.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:08 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:09.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:08 smithi012 ceph-mon[95978]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:01:09.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:08 smithi012 ceph-mon[95978]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:01:09.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:08 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:09.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:08 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T05:01:09.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:08 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T05:01:09.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:08 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:09.744 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:09 smithi012 ceph-mon[91852]: Upgrade: Updating mon.b 2024-04-26T05:01:09.745 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:09 smithi012 ceph-mon[91852]: Deploying daemon mon.b on smithi116 2024-04-26T05:01:09.745 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:09 smithi012 ceph-mon[91852]: pgmap v58: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:09.745 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:09 smithi012 ceph-mon[95978]: Upgrade: Updating mon.b 2024-04-26T05:01:09.745 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:09 smithi012 ceph-mon[95978]: Deploying daemon mon.b on smithi116 2024-04-26T05:01:09.745 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:09 smithi012 ceph-mon[95978]: pgmap v58: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:09.891 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:09 smithi116 ceph-mon[29897]: Upgrade: Updating mon.b 2024-04-26T05:01:09.891 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:09 smithi116 ceph-mon[29897]: Deploying daemon mon.b on smithi116 2024-04-26T05:01:09.891 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:09 smithi116 ceph-mon[29897]: pgmap v58: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:10.241 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:01:10 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:01:10.087Z caller=alerting.go:366 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephMonDown: template: __alert_CephMonDown:1:427: executing \"__alert_CephMonDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"mon.a\"} on the right hand-side of the operation: [{ceph_daemon=\"mon.a\", ceph_version=\"ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", public_addr=\"172.21.15.12\", rank=\"0\"}, {ceph_daemon=\"mon.a\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", hostname=\"smithi012\", instance=\"smithi116.front.sepia.ceph.com:9283\", job=\"ceph\", public_addr=\"172.21.15.12\", rank=\"0\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-04-26T05:01:10.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:09 smithi116 systemd[1]: Stopping Ceph mon.b for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T05:01:10.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:10 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-b[29893]: 2024-04-26T05:01:10.132+0000 7fbeb818b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:01:10.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:10 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-b[29893]: 2024-04-26T05:01:10.132+0000 7fbeb818b700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-04-26T05:01:10.508 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:10 smithi116 podman[85335]: 2024-04-26 05:01:10.255728963 +0000 UTC m=+0.221891976 container died e6cd16f014178dae87b9d89437299336c95137420d1ea34ca296a9beeede4960 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-b, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=quincy-c61472c, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424) 2024-04-26T05:01:10.775 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:10 smithi116 podman[85335]: 2024-04-26 05:01:10.58249869 +0000 UTC m=+0.548661701 container cleanup e6cd16f014178dae87b9d89437299336c95137420d1ea34ca296a9beeede4960 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-b, org.label-schema.build-date=20240424, RELEASE=quincy-c61472c, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-04-26T05:01:10.776 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:10 smithi116 bash[85335]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-b 2024-04-26T05:01:11.204 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:10 smithi116 podman[85349]: 2024-04-26 05:01:10.775438938 +0000 UTC m=+0.521129720 container remove e6cd16f014178dae87b9d89437299336c95137420d1ea34ca296a9beeede4960 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-b, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-c61472c, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-04-26T05:01:11.204 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mon.b.service: Deactivated successfully. 2024-04-26T05:01:11.204 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 systemd[1]: Stopped Ceph mon.b for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T05:01:11.204 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mon.b.service: Consumed 8.547s CPU time. 2024-04-26T05:01:11.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 systemd[1]: Starting Ceph mon.b for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T05:01:11.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 podman[85483]: 2024-04-26 05:01:11.324756603 +0000 UTC m=+0.018804248 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:01:11.821 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 podman[85483]: 2024-04-26 05:01:11.568573873 +0000 UTC m=+0.262621518 container create a0c4cbd180dca3a01b2950b2898656630d35f42c2a80b12bb4179d5ece61178f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-b, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb) 2024-04-26T05:01:11.855 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 podman[85483]: 2024-04-26 05:01:11.821260491 +0000 UTC m=+0.515308135 container init a0c4cbd180dca3a01b2950b2898656630d35f42c2a80b12bb4179d5ece61178f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-b, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-04-26T05:01:11.855 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 podman[85483]: 2024-04-26 05:01:11.824410832 +0000 UTC m=+0.518458486 container start a0c4cbd180dca3a01b2950b2898656630d35f42c2a80b12bb4179d5ece61178f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-b, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-04-26T05:01:11.855 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: set uid:gid to 167:167 (ceph:ceph) 2024-04-26T05:01:11.856 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev), process ceph-mon, pid 2 2024-04-26T05:01:11.856 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: pidfile_write: ignore empty --pid-file 2024-04-26T05:01:11.856 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: load: jerasure load: lrc 2024-04-26T05:01:11.856 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: RocksDB version: 7.9.2 2024-04-26T05:01:11.856 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Git sha 0 2024-04-26T05:01:11.856 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Compile date 2024-04-25 16:43:49 2024-04-26T05:01:11.856 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: DB SUMMARY 2024-04-26T05:01:11.856 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: DB Session ID: QJY37NQAKMWPUD9NQVS8 2024-04-26T05:01:11.856 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: CURRENT file: CURRENT 2024-04-26T05:01:11.856 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: IDENTITY file: IDENTITY 2024-04-26T05:01:11.856 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1084 Bytes 2024-04-26T05:01:11.856 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2024-04-26T05:01:11.856 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 1859446 ; 2024-04-26T05:01:11.856 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.error_if_exists: 0 2024-04-26T05:01:11.856 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.create_if_missing: 0 2024-04-26T05:01:11.857 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.paranoid_checks: 1 2024-04-26T05:01:11.857 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-26T05:01:11.857 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-26T05:01:11.857 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-26T05:01:12.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.env: 0x56511be02c60 2024-04-26T05:01:12.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.fs: PosixFileSystem 2024-04-26T05:01:12.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.info_log: 0x56511c67daa0 2024-04-26T05:01:12.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.max_file_opening_threads: 16 2024-04-26T05:01:12.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.statistics: (nil) 2024-04-26T05:01:12.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.use_fsync: 0 2024-04-26T05:01:12.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.max_log_file_size: 0 2024-04-26T05:01:12.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-26T05:01:12.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-26T05:01:12.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.keep_log_file_num: 1000 2024-04-26T05:01:12.243 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.recycle_log_file_num: 0 2024-04-26T05:01:12.244 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.allow_fallocate: 1 2024-04-26T05:01:12.244 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.allow_mmap_reads: 0 2024-04-26T05:01:12.244 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.allow_mmap_writes: 0 2024-04-26T05:01:12.244 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.use_direct_reads: 0 2024-04-26T05:01:12.244 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T05:01:12.244 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T05:01:12.245 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.db_log_dir: 2024-04-26T05:01:12.245 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.wal_dir: 2024-04-26T05:01:12.245 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T05:01:12.245 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T05:01:12.245 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T05:01:12.245 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T05:01:12.245 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T05:01:12.245 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T05:01:12.245 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T05:01:12.245 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T05:01:12.245 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.write_buffer_manager: 0x56511c650820 2024-04-26T05:01:12.245 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T05:01:12.245 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T05:01:12.246 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T05:01:12.246 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T05:01:12.246 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T05:01:12.246 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T05:01:12.246 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T05:01:12.246 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T05:01:12.246 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.unordered_write: 0 2024-04-26T05:01:12.246 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T05:01:12.247 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T05:01:12.248 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T05:01:12.248 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T05:01:12.248 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.row_cache: None 2024-04-26T05:01:12.248 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.wal_filter: None 2024-04-26T05:01:12.248 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T05:01:12.248 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T05:01:12.248 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.two_write_queues: 0 2024-04-26T05:01:12.249 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T05:01:12.249 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.wal_compression: 0 2024-04-26T05:01:12.249 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.atomic_flush: 0 2024-04-26T05:01:12.249 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T05:01:12.249 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T05:01:12.249 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T05:01:12.249 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.log_readahead_size: 0 2024-04-26T05:01:12.249 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T05:01:12.249 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T05:01:12.249 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T05:01:12.250 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T05:01:12.250 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T05:01:12.250 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T05:01:12.250 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-26T05:01:12.250 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.max_background_jobs: 2 2024-04-26T05:01:12.250 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.max_background_compactions: -1 2024-04-26T05:01:12.251 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.max_subcompactions: 1 2024-04-26T05:01:12.251 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T05:01:12.251 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T05:01:12.251 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T05:01:12.251 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T05:01:12.251 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T05:01:12.251 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T05:01:12.251 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T05:01:12.251 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T05:01:12.251 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.max_open_files: -1 2024-04-26T05:01:12.251 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T05:01:12.251 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T05:01:12.252 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T05:01:12.252 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T05:01:12.252 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.max_background_flushes: -1 2024-04-26T05:01:12.252 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Compression algorithms supported: 2024-04-26T05:01:12.252 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T05:01:12.252 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: kZSTD supported: 0 2024-04-26T05:01:12.252 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: kXpressCompression supported: 0 2024-04-26T05:01:12.252 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T05:01:12.252 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: kZlibCompression supported: 1 2024-04-26T05:01:12.252 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: kSnappyCompression supported: 1 2024-04-26T05:01:12.252 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: kLZ4Compression supported: 1 2024-04-26T05:01:12.252 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: kBZip2Compression supported: 0 2024-04-26T05:01:12.252 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T05:01:12.253 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-26T05:01:12.253 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-04-26T05:01:12.253 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-26T05:01:12.253 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T05:01:12.253 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.merge_operator: 2024-04-26T05:01:12.253 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.compaction_filter: None 2024-04-26T05:01:12.253 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T05:01:12.253 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T05:01:12.253 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T05:01:12.253 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T05:01:12.253 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56511c67dba0) 2024-04-26T05:01:12.253 INFO:journalctl@ceph.mon.b.smithi116.stdout: cache_index_and_filter_blocks: 1 2024-04-26T05:01:12.253 INFO:journalctl@ceph.mon.b.smithi116.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T05:01:12.253 INFO:journalctl@ceph.mon.b.smithi116.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T05:01:12.253 INFO:journalctl@ceph.mon.b.smithi116.stdout: pin_top_level_index_and_filter: 1 2024-04-26T05:01:12.254 INFO:journalctl@ceph.mon.b.smithi116.stdout: index_type: 0 2024-04-26T05:01:12.254 INFO:journalctl@ceph.mon.b.smithi116.stdout: data_block_index_type: 0 2024-04-26T05:01:12.254 INFO:journalctl@ceph.mon.b.smithi116.stdout: index_shortening: 1 2024-04-26T05:01:12.254 INFO:journalctl@ceph.mon.b.smithi116.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T05:01:12.254 INFO:journalctl@ceph.mon.b.smithi116.stdout: checksum: 4 2024-04-26T05:01:12.254 INFO:journalctl@ceph.mon.b.smithi116.stdout: no_block_cache: 0 2024-04-26T05:01:12.254 INFO:journalctl@ceph.mon.b.smithi116.stdout: block_cache: 0x56511c681090 2024-04-26T05:01:12.254 INFO:journalctl@ceph.mon.b.smithi116.stdout: block_cache_name: BinnedLRUCache 2024-04-26T05:01:12.254 INFO:journalctl@ceph.mon.b.smithi116.stdout: block_cache_options: 2024-04-26T05:01:12.254 INFO:journalctl@ceph.mon.b.smithi116.stdout: capacity : 536870912 2024-04-26T05:01:12.254 INFO:journalctl@ceph.mon.b.smithi116.stdout: num_shard_bits : 4 2024-04-26T05:01:12.254 INFO:journalctl@ceph.mon.b.smithi116.stdout: strict_capacity_limit : 0 2024-04-26T05:01:12.254 INFO:journalctl@ceph.mon.b.smithi116.stdout: high_pri_pool_ratio: 0.000 2024-04-26T05:01:12.255 INFO:journalctl@ceph.mon.b.smithi116.stdout: block_cache_compressed: (nil) 2024-04-26T05:01:12.255 INFO:journalctl@ceph.mon.b.smithi116.stdout: persistent_cache: (nil) 2024-04-26T05:01:12.255 INFO:journalctl@ceph.mon.b.smithi116.stdout: block_size: 4096 2024-04-26T05:01:12.255 INFO:journalctl@ceph.mon.b.smithi116.stdout: block_size_deviation: 10 2024-04-26T05:01:12.255 INFO:journalctl@ceph.mon.b.smithi116.stdout: block_restart_interval: 16 2024-04-26T05:01:12.255 INFO:journalctl@ceph.mon.b.smithi116.stdout: index_block_restart_interval: 1 2024-04-26T05:01:12.255 INFO:journalctl@ceph.mon.b.smithi116.stdout: metadata_block_size: 4096 2024-04-26T05:01:12.255 INFO:journalctl@ceph.mon.b.smithi116.stdout: partition_filters: 0 2024-04-26T05:01:12.255 INFO:journalctl@ceph.mon.b.smithi116.stdout: use_delta_encoding: 1 2024-04-26T05:01:12.255 INFO:journalctl@ceph.mon.b.smithi116.stdout: filter_policy: bloomfilter 2024-04-26T05:01:12.255 INFO:journalctl@ceph.mon.b.smithi116.stdout: whole_key_filtering: 1 2024-04-26T05:01:12.255 INFO:journalctl@ceph.mon.b.smithi116.stdout: verify_compression: 0 2024-04-26T05:01:12.255 INFO:journalctl@ceph.mon.b.smithi116.stdout: read_amp_bytes_per_bit: 0 2024-04-26T05:01:12.256 INFO:journalctl@ceph.mon.b.smithi116.stdout: format_version: 5 2024-04-26T05:01:12.256 INFO:journalctl@ceph.mon.b.smithi116.stdout: enable_index_compression: 1 2024-04-26T05:01:12.256 INFO:journalctl@ceph.mon.b.smithi116.stdout: block_align: 0 2024-04-26T05:01:12.256 INFO:journalctl@ceph.mon.b.smithi116.stdout: max_auto_readahead_size: 262144 2024-04-26T05:01:12.256 INFO:journalctl@ceph.mon.b.smithi116.stdout: prepopulate_block_cache: 0 2024-04-26T05:01:12.256 INFO:journalctl@ceph.mon.b.smithi116.stdout: initial_auto_readahead_size: 8192 2024-04-26T05:01:12.256 INFO:journalctl@ceph.mon.b.smithi116.stdout: num_file_reads_for_auto_readahead: 2 2024-04-26T05:01:12.256 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T05:01:12.256 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T05:01:12.256 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.compression: NoCompression 2024-04-26T05:01:12.256 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T05:01:12.256 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T05:01:12.256 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T05:01:12.256 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.num_levels: 7 2024-04-26T05:01:12.256 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T05:01:12.257 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T05:01:12.257 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T05:01:12.257 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T05:01:12.257 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T05:01:12.257 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T05:01:12.257 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T05:01:12.257 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T05:01:12.257 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T05:01:12.257 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T05:01:12.257 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-26T05:01:12.257 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-26T05:01:12.257 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T05:01:12.257 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T05:01:12.258 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T05:01:12.258 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T05:01:12.258 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T05:01:12.258 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-26T05:01:12.258 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T05:01:12.258 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T05:01:12.258 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-26T05:01:12.258 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T05:01:12.258 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T05:01:12.258 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T05:01:12.258 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T05:01:12.258 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T05:01:12.259 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T05:01:12.259 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T05:01:12.259 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T05:01:12.259 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T05:01:12.259 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T05:01:12.259 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T05:01:12.259 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T05:01:12.259 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T05:01:12.259 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T05:01:12.259 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T05:01:12.259 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T05:01:12.259 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T05:01:12.259 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-26T05:01:12.259 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.arena_block_size: 1048576 2024-04-26T05:01:12.259 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T05:01:12.260 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T05:01:12.260 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T05:01:12.260 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T05:01:12.260 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T05:01:12.260 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T05:01:12.260 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T05:01:12.260 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T05:01:12.260 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T05:01:12.260 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T05:01:12.260 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T05:01:12.260 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T05:01:12.260 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T05:01:12.260 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.table_properties_collectors: 2024-04-26T05:01:12.260 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.inplace_update_support: 0 2024-04-26T05:01:12.260 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T05:01:12.261 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T05:01:12.261 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T05:01:12.261 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T05:01:12.261 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.bloom_locality: 0 2024-04-26T05:01:12.261 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.max_successive_merges: 0 2024-04-26T05:01:12.261 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T05:01:12.261 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T05:01:12.261 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T05:01:12.261 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T05:01:12.261 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.ttl: 2592000 2024-04-26T05:01:12.261 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T05:01:12.262 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-26T05:01:12.262 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-26T05:01:12.262 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.enable_blob_files: false 2024-04-26T05:01:12.262 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.min_blob_size: 0 2024-04-26T05:01:12.262 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T05:01:12.262 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T05:01:12.262 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T05:01:12.262 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T05:01:12.262 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-26T05:01:12.262 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-26T05:01:12.262 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.blob_file_starting_level: 0 2024-04-26T05:01:12.262 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-26T05:01:12.262 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-04-26T05:01:12.263 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T05:01:12.263 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13043, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T05:01:12.263 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-04-26T05:01:12.263 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 1bb3f39f-47da-4dfd-be5b-6e12535c1a94 2024-04-26T05:01:12.263 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714107671853988, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-04-26T05:01:12.263 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-04-26T05:01:12.263 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714107671861145, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 1148757, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13048, "largest_seqno": 14185, "table_properties": {"data_size": 1143594, "index_size": 3060, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1093, "raw_key_size": 9669, "raw_average_key_size": 22, "raw_value_size": 1134120, "raw_average_value_size": 2637, "num_data_blocks": 139, "num_entries": 430, "num_filter_entries": 430, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1714107671, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "1bb3f39f-47da-4dfd-be5b-6e12535c1a94", "db_session_id": "QJY37NQAKMWPUD9NQVS8", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-04-26T05:01:12.263 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714107671861267, "job": 1, "event": "recovery_finished"} 2024-04-26T05:01:12.263 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-04-26T05:01:12.263 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T05:01:12.263 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T05:01:12.263 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x56511c772000 2024-04-26T05:01:12.263 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: DB pointer 0x56511c75c000 2024-04-26T05:01:12.263 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-26T05:01:12.264 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-26T05:01:12.264 INFO:journalctl@ceph.mon.b.smithi116.stdout: ** DB Stats ** 2024-04-26T05:01:12.264 INFO:journalctl@ceph.mon.b.smithi116.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T05:01:12.264 INFO:journalctl@ceph.mon.b.smithi116.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T05:01:12.264 INFO:journalctl@ceph.mon.b.smithi116.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T05:01:12.264 INFO:journalctl@ceph.mon.b.smithi116.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T05:01:12.264 INFO:journalctl@ceph.mon.b.smithi116.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T05:01:12.264 INFO:journalctl@ceph.mon.b.smithi116.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T05:01:12.264 INFO:journalctl@ceph.mon.b.smithi116.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T05:01:12.264 INFO:journalctl@ceph.mon.b.smithi116.stdout: 2024-04-26T05:01:12.264 INFO:journalctl@ceph.mon.b.smithi116.stdout: ** Compaction Stats [default] ** 2024-04-26T05:01:12.264 INFO:journalctl@ceph.mon.b.smithi116.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T05:01:12.264 INFO:journalctl@ceph.mon.b.smithi116.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-26T05:01:12.264 INFO:journalctl@ceph.mon.b.smithi116.stdout: L0 1/0 1.10 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 336.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-26T05:01:12.264 INFO:journalctl@ceph.mon.b.smithi116.stdout: L6 1/0 8.50 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-26T05:01:12.265 INFO:journalctl@ceph.mon.b.smithi116.stdout: Sum 2/0 9.60 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 336.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-26T05:01:12.265 INFO:journalctl@ceph.mon.b.smithi116.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 336.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-26T05:01:12.265 INFO:journalctl@ceph.mon.b.smithi116.stdout: 2024-04-26T05:01:12.265 INFO:journalctl@ceph.mon.b.smithi116.stdout: ** Compaction Stats [default] ** 2024-04-26T05:01:12.265 INFO:journalctl@ceph.mon.b.smithi116.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T05:01:12.265 INFO:journalctl@ceph.mon.b.smithi116.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T05:01:12.265 INFO:journalctl@ceph.mon.b.smithi116.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 336.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-26T05:01:12.265 INFO:journalctl@ceph.mon.b.smithi116.stdout: 2024-04-26T05:01:12.265 INFO:journalctl@ceph.mon.b.smithi116.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-26T05:01:12.265 INFO:journalctl@ceph.mon.b.smithi116.stdout: 2024-04-26T05:01:12.265 INFO:journalctl@ceph.mon.b.smithi116.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T05:01:12.265 INFO:journalctl@ceph.mon.b.smithi116.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-04-26T05:01:12.265 INFO:journalctl@ceph.mon.b.smithi116.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T05:01:12.265 INFO:journalctl@ceph.mon.b.smithi116.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T05:01:12.265 INFO:journalctl@ceph.mon.b.smithi116.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T05:01:12.266 INFO:journalctl@ceph.mon.b.smithi116.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T05:01:12.266 INFO:journalctl@ceph.mon.b.smithi116.stdout: Cumulative compaction: 0.00 GB write, 114.29 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T05:01:12.266 INFO:journalctl@ceph.mon.b.smithi116.stdout: Interval compaction: 0.00 GB write, 114.29 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T05:01:12.266 INFO:journalctl@ceph.mon.b.smithi116.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T05:01:12.266 INFO:journalctl@ceph.mon.b.smithi116.stdout: Block cache BinnedLRUCache@0x56511c681090#2 capacity: 512.00 MB usage: 4.23 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-04-26T05:01:12.266 INFO:journalctl@ceph.mon.b.smithi116.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.16 KB,0.000220537%) IndexBlock(1,3.08 KB,0.000587106%) Misc(1,0.00 KB,0%) 2024-04-26T05:01:12.266 INFO:journalctl@ceph.mon.b.smithi116.stdout: 2024-04-26T05:01:12.266 INFO:journalctl@ceph.mon.b.smithi116.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T05:01:12.266 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: starting mon.b rank 1 at public addrs [v2:172.21.15.116:3300/0,v1:172.21.15.116:6789/0] at bind addrs [v2:172.21.15.116:3300/0,v1:172.21.15.116:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T05:01:12.266 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: mon.b@-1(???) e3 preinit fsid 0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T05:01:12.266 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: mon.b@-1(???).mds e9 new map 2024-04-26T05:01:12.266 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: mon.b@-1(???).mds e9 print_map 2024-04-26T05:01:12.266 INFO:journalctl@ceph.mon.b.smithi116.stdout: e9 2024-04-26T05:01:12.266 INFO:journalctl@ceph.mon.b.smithi116.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T05:01:12.266 INFO:journalctl@ceph.mon.b.smithi116.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T05:01:12.267 INFO:journalctl@ceph.mon.b.smithi116.stdout: legacy client fscid: 1 2024-04-26T05:01:12.267 INFO:journalctl@ceph.mon.b.smithi116.stdout: 2024-04-26T05:01:12.267 INFO:journalctl@ceph.mon.b.smithi116.stdout: Filesystem 'foo' (1) 2024-04-26T05:01:12.267 INFO:journalctl@ceph.mon.b.smithi116.stdout: fs_name foo 2024-04-26T05:01:12.267 INFO:journalctl@ceph.mon.b.smithi116.stdout: epoch 8 2024-04-26T05:01:12.267 INFO:journalctl@ceph.mon.b.smithi116.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-26T05:01:12.267 INFO:journalctl@ceph.mon.b.smithi116.stdout: created 2024-04-26T04:55:07.451848+0000 2024-04-26T05:01:12.267 INFO:journalctl@ceph.mon.b.smithi116.stdout: modified 2024-04-26T04:55:21.872954+0000 2024-04-26T05:01:12.267 INFO:journalctl@ceph.mon.b.smithi116.stdout: tableserver 0 2024-04-26T05:01:12.267 INFO:journalctl@ceph.mon.b.smithi116.stdout: root 0 2024-04-26T05:01:12.267 INFO:journalctl@ceph.mon.b.smithi116.stdout: session_timeout 60 2024-04-26T05:01:12.268 INFO:journalctl@ceph.mon.b.smithi116.stdout: session_autoclose 300 2024-04-26T05:01:12.268 INFO:journalctl@ceph.mon.b.smithi116.stdout: max_file_size 1099511627776 2024-04-26T05:01:12.268 INFO:journalctl@ceph.mon.b.smithi116.stdout: max_xattr_size 65536 2024-04-26T05:01:12.268 INFO:journalctl@ceph.mon.b.smithi116.stdout: required_client_features {} 2024-04-26T05:01:12.268 INFO:journalctl@ceph.mon.b.smithi116.stdout: last_failure 0 2024-04-26T05:01:12.268 INFO:journalctl@ceph.mon.b.smithi116.stdout: last_failure_osd_epoch 0 2024-04-26T05:01:12.268 INFO:journalctl@ceph.mon.b.smithi116.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T05:01:12.268 INFO:journalctl@ceph.mon.b.smithi116.stdout: max_mds 1 2024-04-26T05:01:12.268 INFO:journalctl@ceph.mon.b.smithi116.stdout: in 0 2024-04-26T05:01:12.268 INFO:journalctl@ceph.mon.b.smithi116.stdout: up {0=14664} 2024-04-26T05:01:12.269 INFO:journalctl@ceph.mon.b.smithi116.stdout: failed 2024-04-26T05:01:12.269 INFO:journalctl@ceph.mon.b.smithi116.stdout: damaged 2024-04-26T05:01:12.269 INFO:journalctl@ceph.mon.b.smithi116.stdout: stopped 2024-04-26T05:01:12.269 INFO:journalctl@ceph.mon.b.smithi116.stdout: data_pools [4] 2024-04-26T05:01:12.269 INFO:journalctl@ceph.mon.b.smithi116.stdout: metadata_pool 3 2024-04-26T05:01:12.269 INFO:journalctl@ceph.mon.b.smithi116.stdout: inline_data disabled 2024-04-26T05:01:12.269 INFO:journalctl@ceph.mon.b.smithi116.stdout: balancer 2024-04-26T05:01:12.269 INFO:journalctl@ceph.mon.b.smithi116.stdout: bal_rank_mask -1 2024-04-26T05:01:12.269 INFO:journalctl@ceph.mon.b.smithi116.stdout: standby_count_wanted 1 2024-04-26T05:01:12.269 INFO:journalctl@ceph.mon.b.smithi116.stdout: qdb_cluster leader: 0 members: 2024-04-26T05:01:12.269 INFO:journalctl@ceph.mon.b.smithi116.stdout: [mds.foo.smithi012.fdqwmo{0:14664} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.12:6834/963323745,v1:172.21.15.12:6835/963323745] compat {c=[1],r=[1],i=[7ff]}] 2024-04-26T05:01:12.269 INFO:journalctl@ceph.mon.b.smithi116.stdout: 2024-04-26T05:01:12.269 INFO:journalctl@ceph.mon.b.smithi116.stdout: 2024-04-26T05:01:12.269 INFO:journalctl@ceph.mon.b.smithi116.stdout: Standby daemons: 2024-04-26T05:01:12.269 INFO:journalctl@ceph.mon.b.smithi116.stdout: 2024-04-26T05:01:12.270 INFO:journalctl@ceph.mon.b.smithi116.stdout: [mds.foo.smithi116.kduyef{-1:24520} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.116:6832/529792701,v1:172.21.15.116:6833/529792701] compat {c=[1],r=[1],i=[7ff]}] 2024-04-26T05:01:12.270 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: mon.b@-1(???).osd e80 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-26T05:01:12.270 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: mon.b@-1(???).osd e80 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T05:01:12.270 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: mon.b@-1(???).osd e80 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T05:01:12.270 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: mon.b@-1(???).osd e80 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T05:01:12.270 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 ceph-mon[85502]: mon.b@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2024-04-26T05:01:12.270 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 bash[85483]: a0c4cbd180dca3a01b2950b2898656630d35f42c2a80b12bb4179d5ece61178f 2024-04-26T05:01:12.270 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:11 smithi116 systemd[1]: Started Ceph mon.b for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T05:01:17.207 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:01:16 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T05:01:16.869+0000 7f0e1e4f1700 -1 mgr.server handle_report got status from non-daemon mon.b 2024-04-26T05:01:17.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:17 smithi116 ceph-mon[85502]: pgmap v60: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:17.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:17 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T05:01:17.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:17 smithi116 ceph-mon[85502]: mon.a calling monitor election 2024-04-26T05:01:17.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:17 smithi116 ceph-mon[85502]: mon.b calling monitor election 2024-04-26T05:01:17.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:17 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T05:01:17.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:17 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T05:01:17.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:17 smithi116 ceph-mon[85502]: pgmap v61: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:17.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:17 smithi116 ceph-mon[85502]: pgmap v62: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:17.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:17 smithi116 ceph-mon[85502]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-26T05:01:17.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:17 smithi116 ceph-mon[85502]: monmap epoch 4 2024-04-26T05:01:17.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:17 smithi116 ceph-mon[85502]: fsid 0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T05:01:17.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:17 smithi116 ceph-mon[85502]: last_changed 2024-04-26T05:01:12.135809+0000 2024-04-26T05:01:17.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:17 smithi116 ceph-mon[85502]: created 2024-04-26T04:48:15.190556+0000 2024-04-26T05:01:17.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:17 smithi116 ceph-mon[85502]: min_mon_release 19 (squid) 2024-04-26T05:01:17.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:17 smithi116 ceph-mon[85502]: election_strategy: 1 2024-04-26T05:01:17.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:17 smithi116 ceph-mon[85502]: 0: [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] mon.a 2024-04-26T05:01:17.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:17 smithi116 ceph-mon[85502]: 1: [v2:172.21.15.116:3300/0,v1:172.21.15.116:6789/0] mon.b 2024-04-26T05:01:17.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:17 smithi116 ceph-mon[85502]: 2: [v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0] mon.c 2024-04-26T05:01:17.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:17 smithi116 ceph-mon[85502]: fsmap foo:1 {0=foo.smithi012.fdqwmo=up:active} 1 up:standby 2024-04-26T05:01:17.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:17 smithi116 ceph-mon[85502]: osdmap e80: 8 total, 8 up, 8 in 2024-04-26T05:01:17.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:17 smithi116 ceph-mon[85502]: mgrmap e39: y(active, since 111s), standbys: x 2024-04-26T05:01:17.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:17 smithi116 ceph-mon[85502]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-04-26T05:01:17.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:17 smithi116 ceph-mon[85502]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); 1/3 mons down, quorum a,b 2024-04-26T05:01:17.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:17 smithi116 ceph-mon[85502]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T05:01:17.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:17 smithi116 ceph-mon[85502]: daemon grafana.a on smithi116 is in error state 2024-04-26T05:01:17.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:17 smithi116 ceph-mon[85502]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-04-26T05:01:17.494 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:17 smithi116 ceph-mon[85502]: mon.c (rank 2) addr [v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0] is down (out of quorum) 2024-04-26T05:01:17.494 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:17 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:17.494 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:17 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:17.494 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:17 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:17.494 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:17 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:17.494 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:17 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:01:17.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:17 smithi012 ceph-mon[91852]: pgmap v60: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:17.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:17 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T05:01:17.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:17 smithi012 ceph-mon[91852]: mon.a calling monitor election 2024-04-26T05:01:17.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:17 smithi012 ceph-mon[91852]: mon.b calling monitor election 2024-04-26T05:01:17.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:17 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T05:01:17.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:17 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T05:01:17.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:17 smithi012 ceph-mon[91852]: pgmap v61: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:17.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:17 smithi012 ceph-mon[91852]: pgmap v62: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:17.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:17 smithi012 ceph-mon[91852]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-26T05:01:17.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:17 smithi012 ceph-mon[91852]: monmap epoch 4 2024-04-26T05:01:17.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:17 smithi012 ceph-mon[91852]: fsid 0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T05:01:17.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:17 smithi012 ceph-mon[91852]: last_changed 2024-04-26T05:01:12.135809+0000 2024-04-26T05:01:17.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:17 smithi012 ceph-mon[91852]: created 2024-04-26T04:48:15.190556+0000 2024-04-26T05:01:17.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:17 smithi012 ceph-mon[91852]: min_mon_release 19 (squid) 2024-04-26T05:01:17.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:17 smithi012 ceph-mon[91852]: election_strategy: 1 2024-04-26T05:01:17.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:17 smithi012 ceph-mon[91852]: 0: [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] mon.a 2024-04-26T05:01:17.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:17 smithi012 ceph-mon[91852]: 1: [v2:172.21.15.116:3300/0,v1:172.21.15.116:6789/0] mon.b 2024-04-26T05:01:17.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:17 smithi012 ceph-mon[91852]: 2: [v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0] mon.c 2024-04-26T05:01:17.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:17 smithi012 ceph-mon[91852]: fsmap foo:1 {0=foo.smithi012.fdqwmo=up:active} 1 up:standby 2024-04-26T05:01:17.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:17 smithi012 ceph-mon[91852]: osdmap e80: 8 total, 8 up, 8 in 2024-04-26T05:01:17.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:17 smithi012 ceph-mon[91852]: mgrmap e39: y(active, since 111s), standbys: x 2024-04-26T05:01:17.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:17 smithi012 ceph-mon[91852]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-04-26T05:01:17.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:17 smithi012 ceph-mon[91852]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); 1/3 mons down, quorum a,b 2024-04-26T05:01:17.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:17 smithi012 ceph-mon[91852]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T05:01:17.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:17 smithi012 ceph-mon[91852]: daemon grafana.a on smithi116 is in error state 2024-04-26T05:01:17.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:17 smithi012 ceph-mon[91852]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-04-26T05:01:17.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:17 smithi012 ceph-mon[91852]: mon.c (rank 2) addr [v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0] is down (out of quorum) 2024-04-26T05:01:17.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:17 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:17.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:17 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:17.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:17 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:17.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:17 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:17.515 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:17 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:01:18.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:01:17 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:01:17] "GET /metrics HTTP/1.1" 200 33825 "" "Prometheus/2.43.0" 2024-04-26T05:01:18.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:18 smithi012 ceph-mon[95978]: pgmap v60: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:18.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:18 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T05:01:18.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:18 smithi012 ceph-mon[95978]: mon.a calling monitor election 2024-04-26T05:01:18.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:18 smithi012 ceph-mon[95978]: mon.b calling monitor election 2024-04-26T05:01:18.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:18 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T05:01:18.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:18 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T05:01:18.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:18 smithi012 ceph-mon[95978]: pgmap v61: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:18.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:18 smithi012 ceph-mon[95978]: pgmap v62: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:18.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:18 smithi012 ceph-mon[95978]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-26T05:01:18.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:18 smithi012 ceph-mon[95978]: monmap epoch 4 2024-04-26T05:01:18.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:18 smithi012 ceph-mon[95978]: fsid 0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T05:01:18.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:18 smithi012 ceph-mon[95978]: last_changed 2024-04-26T05:01:12.135809+0000 2024-04-26T05:01:18.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:18 smithi012 ceph-mon[95978]: created 2024-04-26T04:48:15.190556+0000 2024-04-26T05:01:18.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:18 smithi012 ceph-mon[95978]: min_mon_release 19 (squid) 2024-04-26T05:01:18.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:18 smithi012 ceph-mon[95978]: election_strategy: 1 2024-04-26T05:01:18.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:18 smithi012 ceph-mon[95978]: 0: [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] mon.a 2024-04-26T05:01:18.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:18 smithi012 ceph-mon[95978]: 1: [v2:172.21.15.116:3300/0,v1:172.21.15.116:6789/0] mon.b 2024-04-26T05:01:18.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:18 smithi012 ceph-mon[95978]: 2: [v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0] mon.c 2024-04-26T05:01:18.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:18 smithi012 ceph-mon[95978]: fsmap foo:1 {0=foo.smithi012.fdqwmo=up:active} 1 up:standby 2024-04-26T05:01:18.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:18 smithi012 ceph-mon[95978]: osdmap e80: 8 total, 8 up, 8 in 2024-04-26T05:01:18.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:18 smithi012 ceph-mon[95978]: mgrmap e39: y(active, since 111s), standbys: x 2024-04-26T05:01:18.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:18 smithi012 ceph-mon[95978]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-04-26T05:01:18.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:18 smithi012 ceph-mon[95978]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); 1/3 mons down, quorum a,b 2024-04-26T05:01:18.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:18 smithi012 ceph-mon[95978]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T05:01:18.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:18 smithi012 ceph-mon[95978]: daemon grafana.a on smithi116 is in error state 2024-04-26T05:01:18.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:18 smithi012 ceph-mon[95978]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-04-26T05:01:18.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:18 smithi012 ceph-mon[95978]: mon.c (rank 2) addr [v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0] is down (out of quorum) 2024-04-26T05:01:18.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:18 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:18.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:18 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:18.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:18 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:18.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:18 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:18.766 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:18 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:01:18.991 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:01:18 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:01:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi116.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:01:19.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:19 smithi116 ceph-mon[85502]: mon.c calling monitor election 2024-04-26T05:01:19.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:19 smithi116 ceph-mon[85502]: pgmap v63: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:19.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:19 smithi116 ceph-mon[85502]: mon.b calling monitor election 2024-04-26T05:01:19.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:19 smithi116 ceph-mon[85502]: mon.a calling monitor election 2024-04-26T05:01:19.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:19 smithi116 ceph-mon[85502]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T05:01:19.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:19 smithi116 ceph-mon[85502]: monmap epoch 4 2024-04-26T05:01:19.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:19 smithi116 ceph-mon[85502]: fsid 0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T05:01:19.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:19 smithi116 ceph-mon[85502]: last_changed 2024-04-26T05:01:12.135809+0000 2024-04-26T05:01:19.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:19 smithi116 ceph-mon[85502]: created 2024-04-26T04:48:15.190556+0000 2024-04-26T05:01:19.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:19 smithi116 ceph-mon[85502]: min_mon_release 19 (squid) 2024-04-26T05:01:19.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:19 smithi116 ceph-mon[85502]: election_strategy: 1 2024-04-26T05:01:19.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:19 smithi116 ceph-mon[85502]: 0: [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] mon.a 2024-04-26T05:01:19.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:19 smithi116 ceph-mon[85502]: 1: [v2:172.21.15.116:3300/0,v1:172.21.15.116:6789/0] mon.b 2024-04-26T05:01:19.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:19 smithi116 ceph-mon[85502]: 2: [v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0] mon.c 2024-04-26T05:01:19.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:19 smithi116 ceph-mon[85502]: fsmap foo:1 {0=foo.smithi012.fdqwmo=up:active} 1 up:standby 2024-04-26T05:01:19.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:19 smithi116 ceph-mon[85502]: osdmap e80: 8 total, 8 up, 8 in 2024-04-26T05:01:19.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:19 smithi116 ceph-mon[85502]: mgrmap e39: y(active, since 112s), standbys: x 2024-04-26T05:01:19.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:19 smithi116 ceph-mon[85502]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-04-26T05:01:19.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:19 smithi116 ceph-mon[85502]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T05:01:19.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:19 smithi116 ceph-mon[85502]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T05:01:19.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:19 smithi116 ceph-mon[85502]: daemon grafana.a on smithi116 is in error state 2024-04-26T05:01:19.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[91852]: mon.c calling monitor election 2024-04-26T05:01:19.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[91852]: pgmap v63: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:19.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[91852]: mon.b calling monitor election 2024-04-26T05:01:19.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[91852]: mon.a calling monitor election 2024-04-26T05:01:19.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[91852]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T05:01:19.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[91852]: monmap epoch 4 2024-04-26T05:01:19.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[91852]: fsid 0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T05:01:19.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[91852]: last_changed 2024-04-26T05:01:12.135809+0000 2024-04-26T05:01:19.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[91852]: created 2024-04-26T04:48:15.190556+0000 2024-04-26T05:01:19.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[91852]: min_mon_release 19 (squid) 2024-04-26T05:01:19.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[91852]: election_strategy: 1 2024-04-26T05:01:19.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[91852]: 0: [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] mon.a 2024-04-26T05:01:19.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[91852]: 1: [v2:172.21.15.116:3300/0,v1:172.21.15.116:6789/0] mon.b 2024-04-26T05:01:19.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[91852]: 2: [v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0] mon.c 2024-04-26T05:01:19.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[91852]: fsmap foo:1 {0=foo.smithi012.fdqwmo=up:active} 1 up:standby 2024-04-26T05:01:19.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[91852]: osdmap e80: 8 total, 8 up, 8 in 2024-04-26T05:01:19.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[91852]: mgrmap e39: y(active, since 112s), standbys: x 2024-04-26T05:01:19.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[91852]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-04-26T05:01:19.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[91852]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T05:01:19.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[91852]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T05:01:19.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[91852]: daemon grafana.a on smithi116 is in error state 2024-04-26T05:01:19.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[95978]: mon.c calling monitor election 2024-04-26T05:01:19.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[95978]: pgmap v63: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:19.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[95978]: mon.b calling monitor election 2024-04-26T05:01:19.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[95978]: mon.a calling monitor election 2024-04-26T05:01:19.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[95978]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T05:01:19.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[95978]: monmap epoch 4 2024-04-26T05:01:19.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[95978]: fsid 0b9350c6-0388-11ef-bc93-c7b262605968 2024-04-26T05:01:19.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[95978]: last_changed 2024-04-26T05:01:12.135809+0000 2024-04-26T05:01:19.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[95978]: created 2024-04-26T04:48:15.190556+0000 2024-04-26T05:01:19.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[95978]: min_mon_release 19 (squid) 2024-04-26T05:01:19.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[95978]: election_strategy: 1 2024-04-26T05:01:19.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[95978]: 0: [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] mon.a 2024-04-26T05:01:19.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[95978]: 1: [v2:172.21.15.116:3300/0,v1:172.21.15.116:6789/0] mon.b 2024-04-26T05:01:19.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[95978]: 2: [v2:172.21.15.12:3301/0,v1:172.21.15.12:6790/0] mon.c 2024-04-26T05:01:19.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[95978]: fsmap foo:1 {0=foo.smithi012.fdqwmo=up:active} 1 up:standby 2024-04-26T05:01:19.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[95978]: osdmap e80: 8 total, 8 up, 8 in 2024-04-26T05:01:19.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[95978]: mgrmap e39: y(active, since 112s), standbys: x 2024-04-26T05:01:19.516 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[95978]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-04-26T05:01:19.516 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[95978]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T05:01:19.516 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[95978]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T05:01:19.516 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:19 smithi012 ceph-mon[95978]: daemon grafana.a on smithi116 is in error state 2024-04-26T05:01:20.366 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:01:20 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:01:20.086Z caller=alerting.go:366 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephMonDown: template: __alert_CephMonDown:1:427: executing \"__alert_CephMonDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"mon.a\"} on the right hand-side of the operation: [{ceph_daemon=\"mon.a\", ceph_version=\"ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", public_addr=\"172.21.15.12\", rank=\"0\"}, {ceph_daemon=\"mon.a\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", hostname=\"smithi012\", instance=\"smithi116.front.sepia.ceph.com:9283\", job=\"ceph\", public_addr=\"172.21.15.12\", rank=\"0\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-04-26T05:01:22.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:21 smithi116 ceph-mon[85502]: pgmap v64: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:22.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:21 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:22.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:21 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:22.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:21 smithi012 ceph-mon[91852]: pgmap v64: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:22.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:21 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:22.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:21 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:22.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:21 smithi012 ceph-mon[95978]: pgmap v64: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:22.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:21 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:22.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:21 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:23.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:22 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:23.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:22 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:23.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:22 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:23.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:22 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:23.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:22 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:23.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:22 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:24.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:23 smithi116 ceph-mon[85502]: pgmap v65: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:24.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:23 smithi012 ceph-mon[91852]: pgmap v65: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:24.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:23 smithi012 ceph-mon[95978]: pgmap v65: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:26.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:25 smithi012 ceph-mon[91852]: pgmap v66: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:26.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:25 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:26.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:25 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:26.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:25 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:26.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:25 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:01:26.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:25 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:26.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:25 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T05:01:26.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:25 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T05:01:26.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:25 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:26.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:25 smithi012 ceph-mon[95978]: pgmap v66: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:26.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:25 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:26.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:25 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:26.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:25 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:26.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:25 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:01:26.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:25 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:26.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:25 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T05:01:26.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:25 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T05:01:26.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:25 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:26.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:25 smithi116 ceph-mon[85502]: pgmap v66: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:26.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:25 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:26.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:25 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:26.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:25 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:26.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:25 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:01:26.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:25 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:26.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:25 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T05:01:26.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:25 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T05:01:26.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:25 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:27.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:26 smithi012 ceph-mon[95978]: pgmap v67: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:27.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:26 smithi012 ceph-mon[95978]: pgmap v68: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:27.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:26 smithi012 ceph-mon[95978]: Reconfiguring mon.a (monmap changed)... 2024-04-26T05:01:27.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:26 smithi012 ceph-mon[95978]: Reconfiguring daemon mon.a on smithi012 2024-04-26T05:01:27.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:26 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:27.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:26 smithi012 ceph-mon[91852]: pgmap v67: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:27.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:26 smithi012 ceph-mon[91852]: pgmap v68: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:27.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:26 smithi012 ceph-mon[91852]: Reconfiguring mon.a (monmap changed)... 2024-04-26T05:01:27.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:26 smithi012 ceph-mon[91852]: Reconfiguring daemon mon.a on smithi012 2024-04-26T05:01:27.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:26 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:27.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:26 smithi116 ceph-mon[85502]: pgmap v67: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:27.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:26 smithi116 ceph-mon[85502]: pgmap v68: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:27.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:26 smithi116 ceph-mon[85502]: Reconfiguring mon.a (monmap changed)... 2024-04-26T05:01:27.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:26 smithi116 ceph-mon[85502]: Reconfiguring daemon mon.a on smithi012 2024-04-26T05:01:27.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:26 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:28.102 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:01:27 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:01:27] "GET /metrics HTTP/1.1" 200 33824 "" "Prometheus/2.43.0" 2024-04-26T05:01:28.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:28 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:28.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:28 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:28.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:28 smithi116 ceph-mon[85502]: Reconfiguring mgr.y (monmap changed)... 2024-04-26T05:01:28.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:28 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T05:01:28.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:28 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T05:01:28.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:28 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:28.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:28 smithi116 ceph-mon[85502]: Reconfiguring daemon mgr.y on smithi012 2024-04-26T05:01:28.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:28 smithi116 ceph-mon[85502]: pgmap v69: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:28.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:28 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:28.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:28 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:28.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:28 smithi012 ceph-mon[95978]: Reconfiguring mgr.y (monmap changed)... 2024-04-26T05:01:28.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:28 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T05:01:28.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:28 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T05:01:28.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:28 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:28.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:28 smithi012 ceph-mon[95978]: Reconfiguring daemon mgr.y on smithi012 2024-04-26T05:01:28.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:28 smithi012 ceph-mon[95978]: pgmap v69: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:28.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:28 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:28.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:28 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:28.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:28 smithi012 ceph-mon[91852]: Reconfiguring mgr.y (monmap changed)... 2024-04-26T05:01:28.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:28 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T05:01:28.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:28 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T05:01:28.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:28 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:28.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:28 smithi012 ceph-mon[91852]: Reconfiguring daemon mgr.y on smithi012 2024-04-26T05:01:28.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:28 smithi012 ceph-mon[91852]: pgmap v69: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:28.991 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:01:28 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:01:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi116.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:01:30.147 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:29 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:30.147 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:29 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:30.147 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:29 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T05:01:30.147 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:29 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T05:01:30.147 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:29 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:30.148 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:29 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:30.148 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:29 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:30.148 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:29 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T05:01:30.148 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:29 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T05:01:30.148 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:29 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:30.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:29 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:30.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:29 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:30.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:29 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T05:01:30.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:29 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T05:01:30.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:29 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:31.185 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:30 smithi012 ceph-mon[91852]: Reconfiguring mon.c (monmap changed)... 2024-04-26T05:01:31.185 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:30 smithi012 ceph-mon[91852]: Reconfiguring daemon mon.c on smithi012 2024-04-26T05:01:31.185 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:30 smithi012 ceph-mon[91852]: pgmap v70: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:31.185 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:30 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:31.186 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:30 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:31.186 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:30 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T05:01:31.186 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:30 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:31.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:30 smithi012 ceph-mon[95978]: Reconfiguring mon.c (monmap changed)... 2024-04-26T05:01:31.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:30 smithi012 ceph-mon[95978]: Reconfiguring daemon mon.c on smithi012 2024-04-26T05:01:31.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:30 smithi012 ceph-mon[95978]: pgmap v70: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:31.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:30 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:31.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:30 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:31.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:30 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T05:01:31.186 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:30 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:31.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:30 smithi116 ceph-mon[85502]: Reconfiguring mon.c (monmap changed)... 2024-04-26T05:01:31.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:30 smithi116 ceph-mon[85502]: Reconfiguring daemon mon.c on smithi012 2024-04-26T05:01:31.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:30 smithi116 ceph-mon[85502]: pgmap v70: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:31.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:30 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:31.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:30 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:31.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:30 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T05:01:31.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:30 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:32.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:31 smithi116 ceph-mon[85502]: Reconfiguring osd.0 (monmap changed)... 2024-04-26T05:01:32.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:31 smithi116 ceph-mon[85502]: Reconfiguring daemon osd.0 on smithi012 2024-04-26T05:01:32.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:31 smithi012 ceph-mon[91852]: Reconfiguring osd.0 (monmap changed)... 2024-04-26T05:01:32.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:31 smithi012 ceph-mon[91852]: Reconfiguring daemon osd.0 on smithi012 2024-04-26T05:01:32.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:31 smithi012 ceph-mon[95978]: Reconfiguring osd.0 (monmap changed)... 2024-04-26T05:01:32.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:31 smithi012 ceph-mon[95978]: Reconfiguring daemon osd.0 on smithi012 2024-04-26T05:01:33.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:32 smithi012 ceph-mon[91852]: pgmap v71: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:33.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:32 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:33.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:32 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:33.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:32 smithi012 ceph-mon[91852]: Reconfiguring osd.1 (monmap changed)... 2024-04-26T05:01:33.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:32 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T05:01:33.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:32 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:33.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:32 smithi012 ceph-mon[91852]: Reconfiguring daemon osd.1 on smithi012 2024-04-26T05:01:33.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:32 smithi012 ceph-mon[95978]: pgmap v71: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:33.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:32 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:33.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:32 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:33.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:32 smithi012 ceph-mon[95978]: Reconfiguring osd.1 (monmap changed)... 2024-04-26T05:01:33.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:32 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T05:01:33.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:32 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:33.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:32 smithi012 ceph-mon[95978]: Reconfiguring daemon osd.1 on smithi012 2024-04-26T05:01:33.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:32 smithi116 ceph-mon[85502]: pgmap v71: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:33.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:32 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:33.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:32 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:33.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:32 smithi116 ceph-mon[85502]: Reconfiguring osd.1 (monmap changed)... 2024-04-26T05:01:33.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:32 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T05:01:33.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:32 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:33.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:32 smithi116 ceph-mon[85502]: Reconfiguring daemon osd.1 on smithi012 2024-04-26T05:01:35.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:34 smithi012 ceph-mon[91852]: pgmap v72: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:35.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:34 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:35.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:34 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:35.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:34 smithi012 ceph-mon[91852]: Reconfiguring osd.2 (monmap changed)... 2024-04-26T05:01:35.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:34 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T05:01:35.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:34 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:35.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:34 smithi012 ceph-mon[91852]: Reconfiguring daemon osd.2 on smithi012 2024-04-26T05:01:35.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:34 smithi012 ceph-mon[95978]: pgmap v72: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:35.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:34 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:35.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:34 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:35.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:34 smithi012 ceph-mon[95978]: Reconfiguring osd.2 (monmap changed)... 2024-04-26T05:01:35.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:34 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T05:01:35.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:34 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:35.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:34 smithi012 ceph-mon[95978]: Reconfiguring daemon osd.2 on smithi012 2024-04-26T05:01:35.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:34 smithi116 ceph-mon[85502]: pgmap v72: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:35.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:34 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:35.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:34 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:35.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:34 smithi116 ceph-mon[85502]: Reconfiguring osd.2 (monmap changed)... 2024-04-26T05:01:35.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:34 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T05:01:35.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:34 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:35.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:34 smithi116 ceph-mon[85502]: Reconfiguring daemon osd.2 on smithi012 2024-04-26T05:01:36.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:36 smithi116 ceph-mon[85502]: pgmap v73: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:36.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:36 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:36.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:36 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:36.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:36 smithi116 ceph-mon[85502]: Reconfiguring osd.3 (monmap changed)... 2024-04-26T05:01:36.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:36 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T05:01:36.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:36 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:36.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:36 smithi116 ceph-mon[85502]: Reconfiguring daemon osd.3 on smithi012 2024-04-26T05:01:36.761 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:36 smithi012 ceph-mon[91852]: pgmap v73: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:36.761 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:36 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:36.761 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:36 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:36.761 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:36 smithi012 ceph-mon[91852]: Reconfiguring osd.3 (monmap changed)... 2024-04-26T05:01:36.761 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:36 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T05:01:36.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:36 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:36.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:36 smithi012 ceph-mon[91852]: Reconfiguring daemon osd.3 on smithi012 2024-04-26T05:01:36.762 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:36 smithi012 ceph-mon[95978]: pgmap v73: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:36.762 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:36 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:36.762 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:36 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:36.762 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:36 smithi012 ceph-mon[95978]: Reconfiguring osd.3 (monmap changed)... 2024-04-26T05:01:36.762 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:36 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T05:01:36.762 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:36 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:36.762 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:36 smithi012 ceph-mon[95978]: Reconfiguring daemon osd.3 on smithi012 2024-04-26T05:01:38.013 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:01:37 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:01:37] "GET /metrics HTTP/1.1" 200 33824 "" "Prometheus/2.43.0" 2024-04-26T05:01:38.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:38 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:38.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:38 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:38.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:38 smithi116 ceph-mon[85502]: Reconfiguring mds.foo.smithi012.fdqwmo (monmap changed)... 2024-04-26T05:01:38.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:38 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi012.fdqwmo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T05:01:38.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:38 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:38.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:38 smithi116 ceph-mon[85502]: Reconfiguring daemon mds.foo.smithi012.fdqwmo on smithi012 2024-04-26T05:01:38.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:38 smithi116 ceph-mon[85502]: pgmap v74: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:38.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:38 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:38.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:38 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:38.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:38 smithi012 ceph-mon[91852]: Reconfiguring mds.foo.smithi012.fdqwmo (monmap changed)... 2024-04-26T05:01:38.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:38 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi012.fdqwmo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T05:01:38.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:38 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:38.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:38 smithi012 ceph-mon[91852]: Reconfiguring daemon mds.foo.smithi012.fdqwmo on smithi012 2024-04-26T05:01:38.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:38 smithi012 ceph-mon[91852]: pgmap v74: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:38.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:38 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:38.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:38 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:38.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:38 smithi012 ceph-mon[95978]: Reconfiguring mds.foo.smithi012.fdqwmo (monmap changed)... 2024-04-26T05:01:38.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:38 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi012.fdqwmo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T05:01:38.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:38 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:38.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:38 smithi012 ceph-mon[95978]: Reconfiguring daemon mds.foo.smithi012.fdqwmo on smithi012 2024-04-26T05:01:38.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:38 smithi012 ceph-mon[95978]: pgmap v74: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:38.991 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:01:38 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:01:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi116.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:01:40.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:39 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:40.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:39 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:40.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:39 smithi116 ceph-mon[85502]: Reconfiguring mon.b (monmap changed)... 2024-04-26T05:01:40.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:39 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T05:01:40.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:39 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T05:01:40.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:39 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:40.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:39 smithi116 ceph-mon[85502]: Reconfiguring daemon mon.b on smithi116 2024-04-26T05:01:40.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:39 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:40.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:39 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:40.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:39 smithi012 ceph-mon[91852]: Reconfiguring mon.b (monmap changed)... 2024-04-26T05:01:40.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:39 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T05:01:40.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:39 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T05:01:40.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:39 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:40.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:39 smithi012 ceph-mon[91852]: Reconfiguring daemon mon.b on smithi116 2024-04-26T05:01:40.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:39 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:40.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:39 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:40.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:39 smithi012 ceph-mon[95978]: Reconfiguring mon.b (monmap changed)... 2024-04-26T05:01:40.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:39 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T05:01:40.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:39 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T05:01:40.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:39 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:40.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:39 smithi012 ceph-mon[95978]: Reconfiguring daemon mon.b on smithi116 2024-04-26T05:01:41.216 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:40 smithi116 ceph-mon[85502]: pgmap v75: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:41.216 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:40 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:41.216 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:40 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:41.216 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:40 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T05:01:41.216 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:40 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T05:01:41.216 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:40 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:41.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:40 smithi012 ceph-mon[91852]: pgmap v75: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:41.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:40 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:41.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:40 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:41.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:40 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T05:01:41.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:40 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T05:01:41.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:40 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:41.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:40 smithi012 ceph-mon[95978]: pgmap v75: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:41.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:40 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:41.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:40 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:41.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:40 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T05:01:41.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:40 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T05:01:41.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:40 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:42.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:41 smithi116 ceph-mon[85502]: Reconfiguring mgr.x (monmap changed)... 2024-04-26T05:01:42.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:41 smithi116 ceph-mon[85502]: Reconfiguring daemon mgr.x on smithi116 2024-04-26T05:01:42.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:41 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:42.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:41 smithi012 ceph-mon[91852]: Reconfiguring mgr.x (monmap changed)... 2024-04-26T05:01:42.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:41 smithi012 ceph-mon[91852]: Reconfiguring daemon mgr.x on smithi116 2024-04-26T05:01:42.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:41 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:42.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:41 smithi012 ceph-mon[95978]: Reconfiguring mgr.x (monmap changed)... 2024-04-26T05:01:42.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:41 smithi012 ceph-mon[95978]: Reconfiguring daemon mgr.x on smithi116 2024-04-26T05:01:42.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:41 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:43.207 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:42 smithi116 ceph-mon[85502]: pgmap v76: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:43.207 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:42 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:43.207 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:42 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:43.207 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:42 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T05:01:43.207 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:42 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:43.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:42 smithi012 ceph-mon[91852]: pgmap v76: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:43.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:42 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:43.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:42 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:43.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:42 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T05:01:43.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:42 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:43.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:42 smithi012 ceph-mon[95978]: pgmap v76: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:43.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:42 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:43.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:42 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:43.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:42 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T05:01:43.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:42 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:44.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:43 smithi116 ceph-mon[85502]: Reconfiguring osd.4 (monmap changed)... 2024-04-26T05:01:44.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:43 smithi116 ceph-mon[85502]: Reconfiguring daemon osd.4 on smithi116 2024-04-26T05:01:44.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:43 smithi012 ceph-mon[91852]: Reconfiguring osd.4 (monmap changed)... 2024-04-26T05:01:44.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:43 smithi012 ceph-mon[91852]: Reconfiguring daemon osd.4 on smithi116 2024-04-26T05:01:44.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:43 smithi012 ceph-mon[95978]: Reconfiguring osd.4 (monmap changed)... 2024-04-26T05:01:44.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:43 smithi012 ceph-mon[95978]: Reconfiguring daemon osd.4 on smithi116 2024-04-26T05:01:45.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:44 smithi116 ceph-mon[85502]: pgmap v77: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:45.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:44 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:45.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:44 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:45.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:44 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T05:01:45.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:44 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:45.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:44 smithi012 ceph-mon[91852]: pgmap v77: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:45.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:44 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:45.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:44 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:45.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:44 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T05:01:45.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:44 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:45.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:44 smithi012 ceph-mon[95978]: pgmap v77: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:45.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:44 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:45.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:44 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:45.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:44 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T05:01:45.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:44 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:46.188 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:45 smithi116 ceph-mon[85502]: Reconfiguring osd.5 (monmap changed)... 2024-04-26T05:01:46.188 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:45 smithi116 ceph-mon[85502]: Reconfiguring daemon osd.5 on smithi116 2024-04-26T05:01:46.188 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:45 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:46.188 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:45 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:46.189 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:45 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T05:01:46.189 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:45 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:46.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:45 smithi012 ceph-mon[95978]: Reconfiguring osd.5 (monmap changed)... 2024-04-26T05:01:46.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:45 smithi012 ceph-mon[95978]: Reconfiguring daemon osd.5 on smithi116 2024-04-26T05:01:46.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:45 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:46.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:45 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:46.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:45 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T05:01:46.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:45 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:46.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:45 smithi012 ceph-mon[91852]: Reconfiguring osd.5 (monmap changed)... 2024-04-26T05:01:46.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:45 smithi012 ceph-mon[91852]: Reconfiguring daemon osd.5 on smithi116 2024-04-26T05:01:46.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:45 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:46.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:45 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:46.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:45 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T05:01:46.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:45 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:47.164 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:46 smithi012 ceph-mon[91852]: pgmap v78: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:47.164 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:46 smithi012 ceph-mon[91852]: Reconfiguring osd.6 (monmap changed)... 2024-04-26T05:01:47.164 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:46 smithi012 ceph-mon[91852]: Reconfiguring daemon osd.6 on smithi116 2024-04-26T05:01:47.164 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:46 smithi012 ceph-mon[95978]: pgmap v78: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:47.165 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:46 smithi012 ceph-mon[95978]: Reconfiguring osd.6 (monmap changed)... 2024-04-26T05:01:47.165 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:46 smithi012 ceph-mon[95978]: Reconfiguring daemon osd.6 on smithi116 2024-04-26T05:01:47.229 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:46 smithi116 ceph-mon[85502]: pgmap v78: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:47.229 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:46 smithi116 ceph-mon[85502]: Reconfiguring osd.6 (monmap changed)... 2024-04-26T05:01:47.229 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:46 smithi116 ceph-mon[85502]: Reconfiguring daemon osd.6 on smithi116 2024-04-26T05:01:48.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:48 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:48.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:48 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:48.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:48 smithi012 ceph-mon[91852]: Reconfiguring osd.7 (monmap changed)... 2024-04-26T05:01:48.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:48 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T05:01:48.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:48 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:48.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:48 smithi012 ceph-mon[91852]: Reconfiguring daemon osd.7 on smithi116 2024-04-26T05:01:48.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:48 smithi012 ceph-mon[91852]: pgmap v79: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:48.264 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:01:47 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:01:47] "GET /metrics HTTP/1.1" 200 33835 "" "Prometheus/2.43.0" 2024-04-26T05:01:48.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:48 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:48.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:48 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:48.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:48 smithi012 ceph-mon[95978]: Reconfiguring osd.7 (monmap changed)... 2024-04-26T05:01:48.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:48 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T05:01:48.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:48 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:48.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:48 smithi012 ceph-mon[95978]: Reconfiguring daemon osd.7 on smithi116 2024-04-26T05:01:48.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:48 smithi012 ceph-mon[95978]: pgmap v79: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:48.479 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:48 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:48.479 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:48 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:48.480 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:48 smithi116 ceph-mon[85502]: Reconfiguring osd.7 (monmap changed)... 2024-04-26T05:01:48.480 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:48 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T05:01:48.480 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:48 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:48.480 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:48 smithi116 ceph-mon[85502]: Reconfiguring daemon osd.7 on smithi116 2024-04-26T05:01:48.480 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:48 smithi116 ceph-mon[85502]: pgmap v79: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:48.741 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:01:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:01:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi116.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:01:50.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:49 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:50.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:49 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:50.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:49 smithi116 ceph-mon[85502]: Reconfiguring mds.foo.smithi116.kduyef (monmap changed)... 2024-04-26T05:01:50.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:49 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi116.kduyef", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T05:01:50.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:49 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:50.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:49 smithi116 ceph-mon[85502]: Reconfiguring daemon mds.foo.smithi116.kduyef on smithi116 2024-04-26T05:01:50.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:49 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:50.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:49 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:50.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:49 smithi012 ceph-mon[91852]: Reconfiguring mds.foo.smithi116.kduyef (monmap changed)... 2024-04-26T05:01:50.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:49 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi116.kduyef", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T05:01:50.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:49 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:50.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:49 smithi012 ceph-mon[91852]: Reconfiguring daemon mds.foo.smithi116.kduyef on smithi116 2024-04-26T05:01:50.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:49 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:50.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:49 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:50.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:49 smithi012 ceph-mon[95978]: Reconfiguring mds.foo.smithi116.kduyef (monmap changed)... 2024-04-26T05:01:50.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:49 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi116.kduyef", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T05:01:50.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:49 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:50.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:49 smithi012 ceph-mon[95978]: Reconfiguring daemon mds.foo.smithi116.kduyef on smithi116 2024-04-26T05:01:51.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:50 smithi116 ceph-mon[85502]: pgmap v80: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:51.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:50 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:51.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:50 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:51.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:50 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:01:51.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:50 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:51.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:50 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:01:51.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:50 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:51.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:50 smithi012 ceph-mon[91852]: pgmap v80: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:51.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:50 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:51.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:50 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:51.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:50 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:01:51.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:50 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:51.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:50 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:01:51.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:50 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:51.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:50 smithi012 ceph-mon[95978]: pgmap v80: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:51.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:50 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:51.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:50 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:51.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:50 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:01:51.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:50 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:51.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:50 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:01:51.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:50 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:01:52.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:51 smithi116 ceph-mon[85502]: pgmap v81: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:52.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:51 smithi116 ceph-mon[85502]: pgmap v82: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:52.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:51 smithi012 ceph-mon[91852]: pgmap v81: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:52.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:51 smithi012 ceph-mon[91852]: pgmap v82: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:52.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:51 smithi012 ceph-mon[95978]: pgmap v81: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:52.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:51 smithi012 ceph-mon[95978]: pgmap v82: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:54.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:53 smithi116 ceph-mon[85502]: pgmap v83: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:54.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:53 smithi012 ceph-mon[91852]: pgmap v83: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:54.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:53 smithi012 ceph-mon[95978]: pgmap v83: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:56.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:55 smithi116 ceph-mon[85502]: pgmap v84: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:56.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:55 smithi012 ceph-mon[91852]: pgmap v84: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:56.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:55 smithi012 ceph-mon[95978]: pgmap v84: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:57.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:56 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:57.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:56 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:57.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:56 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:58.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:57 smithi116 ceph-mon[85502]: pgmap v85: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:58.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:57 smithi012 ceph-mon[91852]: pgmap v85: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:58.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:57 smithi012 ceph-mon[95978]: pgmap v85: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:58.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:01:57 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:01:57] "GET /metrics HTTP/1.1" 200 33837 "" "Prometheus/2.43.0" 2024-04-26T05:01:58.991 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:01:58 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:01:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi116.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:02:00.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:01:59 smithi116 ceph-mon[85502]: pgmap v86: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:00.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:01:59 smithi012 ceph-mon[91852]: pgmap v86: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:00.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:01:59 smithi012 ceph-mon[95978]: pgmap v86: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:02.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:01 smithi116 ceph-mon[85502]: pgmap v87: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:02.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:01 smithi012 ceph-mon[91852]: pgmap v87: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:02.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:01 smithi012 ceph-mon[95978]: pgmap v87: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:04.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:03 smithi116 ceph-mon[85502]: pgmap v88: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:04.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:03 smithi012 ceph-mon[91852]: pgmap v88: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:04.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:03 smithi012 ceph-mon[95978]: pgmap v88: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:06.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:05 smithi116 ceph-mon[85502]: pgmap v89: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:06.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:05 smithi012 ceph-mon[91852]: pgmap v89: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:06.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:05 smithi012 ceph-mon[95978]: pgmap v89: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:08.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:07 smithi116 ceph-mon[85502]: pgmap v90: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:08.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:07 smithi012 ceph-mon[91852]: pgmap v90: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:08.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:07 smithi012 ceph-mon[95978]: pgmap v90: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:08.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:02:07 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:02:07] "GET /metrics HTTP/1.1" 200 33837 "" "Prometheus/2.43.0" 2024-04-26T05:02:08.992 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:02:08 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:02:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi116.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:02:10.178 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:09 smithi012 ceph-mon[91852]: pgmap v91: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:10.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:09 smithi012 ceph-mon[95978]: pgmap v91: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:10.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:09 smithi116 ceph-mon[85502]: pgmap v91: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:10.610 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade resume' 2024-04-26T05:02:10.884 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:02:12.003 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:11 smithi012 ceph-mon[91852]: pgmap v92: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:12.004 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:11 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:12.004 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:11 smithi012 ceph-mon[95978]: pgmap v92: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:12.004 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:11 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:12.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:11 smithi116 ceph-mon[85502]: pgmap v92: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:12.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:11 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:12.789 INFO:teuthology.orchestra.run.smithi012.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:02:13.444 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'echo wait for all mons to upgrade' 2024-04-26T05:02:14.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:13 smithi012 ceph-mon[91852]: pgmap v93: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:14.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:13 smithi012 ceph-mon[91852]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:02:14.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:13 smithi012 ceph-mon[91852]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:02:14.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:13 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:14.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:13 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:02:14.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:13 smithi012 ceph-mon[95978]: pgmap v93: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:14.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:13 smithi012 ceph-mon[95978]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:02:14.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:13 smithi012 ceph-mon[95978]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:02:14.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:13 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:14.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:13 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:02:14.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:13 smithi116 ceph-mon[85502]: pgmap v93: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:14.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:13 smithi116 ceph-mon[85502]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:02:14.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:13 smithi116 ceph-mon[85502]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:02:14.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:13 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:14.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:13 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:02:14.427 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:02:16.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:15 smithi116 ceph-mon[85502]: pgmap v94: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:16.260 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:15 smithi012 ceph-mon[91852]: pgmap v94: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:16.260 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:15 smithi012 ceph-mon[95978]: pgmap v94: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:16.713 INFO:teuthology.orchestra.run.smithi012.stdout:wait for all mons to upgrade 2024-04-26T05:02:17.930 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:17 smithi012 ceph-mon[91852]: pgmap v95: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:17.930 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:17 smithi012 ceph-mon[95978]: pgmap v95: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:17.931 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2024-04-26T05:02:18.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:17 smithi116 ceph-mon[85502]: pgmap v95: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:18.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:02:17 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:02:17] "GET /metrics HTTP/1.1" 200 33838 "" "Prometheus/2.43.0" 2024-04-26T05:02:18.647 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:02:18.991 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:02:18 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:02:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi116.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:02:19.733 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:19 smithi012 ceph-mon[91852]: pgmap v96: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:19.733 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:19 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:19.733 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:19 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:19.733 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:19 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:19.733 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:19 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:02:19.733 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:19 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:19.734 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:19 smithi012 ceph-mon[95978]: pgmap v96: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:19.734 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:19 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:19.734 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:19 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:19.734 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:19 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:19.734 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:19 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:02:19.734 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:19 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:19.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:19 smithi116 ceph-mon[85502]: pgmap v96: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:19.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:19 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:19.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:19 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:19.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:19 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:19.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:19 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:02:19.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:19 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:20.618 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 3 2024-04-26T05:02:20.732 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[95978]: pgmap v97: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:20.733 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[95978]: pgmap v98: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:20.733 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:02:20.733 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:02:20.733 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:02:20.733 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[95978]: Upgrade: Setting container_image for all mon 2024-04-26T05:02:20.733 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:20.733 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T05:02:20.733 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-26T05:02:20.733 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T05:02:20.733 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-26T05:02:20.733 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T05:02:20.733 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-26T05:02:20.733 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:02:20.734 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[95978]: Upgrade: Setting container_image for all crash 2024-04-26T05:02:20.734 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:20.734 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T05:02:20.734 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[95978]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T05:02:20.734 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[95978]: Upgrade: osd.0 is safe to restart 2024-04-26T05:02:20.734 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/3624823069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:02:20.734 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[91852]: pgmap v97: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:20.734 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[91852]: pgmap v98: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:20.734 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:02:20.734 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:02:20.734 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:02:20.735 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[91852]: Upgrade: Setting container_image for all mon 2024-04-26T05:02:20.735 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:20.735 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T05:02:20.735 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-26T05:02:20.735 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T05:02:20.735 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-26T05:02:20.735 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T05:02:20.735 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-26T05:02:20.735 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:02:20.735 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[91852]: Upgrade: Setting container_image for all crash 2024-04-26T05:02:20.735 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:20.735 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T05:02:20.735 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[91852]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T05:02:20.735 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[91852]: Upgrade: osd.0 is safe to restart 2024-04-26T05:02:20.736 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:20 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/3624823069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:02:20.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:20 smithi116 ceph-mon[85502]: pgmap v97: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:20.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:20 smithi116 ceph-mon[85502]: pgmap v98: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:20.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:20 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:02:20.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:20 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:02:20.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:20 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:02:20.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:20 smithi116 ceph-mon[85502]: Upgrade: Setting container_image for all mon 2024-04-26T05:02:20.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:20 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:20.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:20 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T05:02:20.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:20 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-26T05:02:20.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:20 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T05:02:20.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:20 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-26T05:02:20.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:20 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T05:02:20.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:20 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-26T05:02:20.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:20 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:02:20.993 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:20 smithi116 ceph-mon[85502]: Upgrade: Setting container_image for all crash 2024-04-26T05:02:20.993 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:20 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:20.993 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:20 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T05:02:20.993 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:20 smithi116 ceph-mon[85502]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T05:02:20.993 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:20 smithi116 ceph-mon[85502]: Upgrade: osd.0 is safe to restart 2024-04-26T05:02:20.993 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:20 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/3624823069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T05:02:21.299 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-26T05:02:21.999 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:02:22.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:22 smithi012 ceph-mon[95978]: pgmap v99: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:22.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:22 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:22.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:22 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T05:02:22.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:22 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:22.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:22 smithi012 ceph-mon[91852]: pgmap v99: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:22.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:22 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:22.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:22 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T05:02:22.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:22 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:22.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:22 smithi116 ceph-mon[85502]: pgmap v99: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:22.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:22 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:22.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:22 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T05:02:22.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:22 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:23.720 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:23 smithi012 ceph-mon[91852]: Upgrade: Updating osd.0 2024-04-26T05:02:23.720 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:23 smithi012 ceph-mon[91852]: Deploying daemon osd.0 on smithi012 2024-04-26T05:02:23.720 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:23 smithi012 ceph-mon[95978]: Upgrade: Updating osd.0 2024-04-26T05:02:23.720 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:23 smithi012 ceph-mon[95978]: Deploying daemon osd.0 on smithi012 2024-04-26T05:02:23.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:23 smithi116 ceph-mon[85502]: Upgrade: Updating osd.0 2024-04-26T05:02:23.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:23 smithi116 ceph-mon[85502]: Deploying daemon osd.0 on smithi012 2024-04-26T05:02:24.734 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:24 smithi012 ceph-mon[91852]: pgmap v100: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:24.734 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:24 smithi012 ceph-mon[95978]: pgmap v100: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:24.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:24 smithi116 ceph-mon[85502]: pgmap v100: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:25.267 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T05:02:25.267 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.a smithi012 *:9093,9094 running (4m) 6s ago 8m 23.9M - 0.25.0 c8568f914cd2 217bb19e153b 2024-04-26T05:02:25.267 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.a smithi116 *:3000 error 64s ago 8m - - 2024-04-26T05:02:25.267 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foo.smithi012.fdqwmo smithi012 running (7m) 6s ago 7m 16.7M - 17.2.7-902-gc61472c8 0de407c1b3af 53f6c82f01b4 2024-04-26T05:02:25.267 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foo.smithi116.kduyef smithi116 running (7m) 64s ago 7m 15.7M - 17.2.7-902-gc61472c8 0de407c1b3af 4c0a0b911593 2024-04-26T05:02:25.267 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.x smithi116 *:8443,9283,8765 running (3m) 64s ago 12m 451M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 5499bf989999 2024-04-26T05:02:25.267 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.y smithi012 *:8443,9283,8765 running (3m) 6s ago 14m 503M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 3a46d0031772 2024-04-26T05:02:25.267 INFO:teuthology.orchestra.run.smithi012.stdout:mon.a smithi012 running (3m) 6s ago 14m 43.6M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 db1f873383b0 2024-04-26T05:02:25.268 INFO:teuthology.orchestra.run.smithi012.stdout:mon.b smithi116 running (73s) 64s ago 12m 19.8M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 a0c4cbd180dc 2024-04-26T05:02:25.268 INFO:teuthology.orchestra.run.smithi012.stdout:mon.c smithi012 running (94s) 6s ago 12m 35.4M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 4d75f0bd2f16 2024-04-26T05:02:25.268 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.a smithi012 *:9100 running (4m) 6s ago 7m 17.0M - 1.5.0 0da6a335fe13 6390338999c1 2024-04-26T05:02:25.268 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.b smithi116 *:9100 running (3m) 64s ago 7m 16.6M - 1.5.0 0da6a335fe13 7d0b1e809531 2024-04-26T05:02:25.268 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (11m) 6s ago 11m 48.4M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 25b6843697db 2024-04-26T05:02:25.268 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (11m) 6s ago 11m 47.6M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 8117a7da3ae3 2024-04-26T05:02:25.268 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (11m) 6s ago 11m 53.5M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 432af63060b8 2024-04-26T05:02:25.268 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (10m) 6s ago 10m 47.9M 2481M 17.2.7-902-gc61472c8 0de407c1b3af f65d4289a970 2024-04-26T05:02:25.268 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi116 running (10m) 64s ago 10m 46.9M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 6b0bbf77cb39 2024-04-26T05:02:25.268 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi116 running (9m) 64s ago 9m 51.5M 2480M 17.2.7-902-gc61472c8 0de407c1b3af a551ef15fa7e 2024-04-26T05:02:25.268 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi116 running (9m) 64s ago 9m 51.3M 2480M 17.2.7-902-gc61472c8 0de407c1b3af cdc0eb6f4fa2 2024-04-26T05:02:25.268 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi116 running (9m) 64s ago 9m 51.2M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 3f6689213fda 2024-04-26T05:02:25.268 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.a smithi116 *:9095 running (3m) 64s ago 8m 57.6M - 2.43.0 a07b618ecd1d e9e5b93523ae 2024-04-26T05:02:26.139 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:26 smithi012 systemd[1]: Stopping Ceph osd.0 for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T05:02:26.423 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade pause' 2024-04-26T05:02:26.685 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:26 smithi012 ceph-mon[95978]: from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:02:26.685 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:26 smithi012 ceph-mon[95978]: pgmap v101: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:26.685 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:26 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:26.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:26 smithi012 ceph-mon[91852]: from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:02:26.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:26 smithi012 ceph-mon[91852]: pgmap v101: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:26.686 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:26 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:26.756 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:02:26.983 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:26 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0[41527]: 2024-04-26T05:02:26.682+0000 7f89a5a2f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:02:26.984 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:26 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0[41527]: 2024-04-26T05:02:26.682+0000 7f89a5a2f700 -1 osd.0 80 *** Got signal Terminated *** 2024-04-26T05:02:26.984 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:26 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0[41527]: 2024-04-26T05:02:26.682+0000 7f89a5a2f700 -1 osd.0 80 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:02:26.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:26 smithi116 ceph-mon[85502]: from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:02:26.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:26 smithi116 ceph-mon[85502]: pgmap v101: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:26.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:26 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:27.982 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:27 smithi012 ceph-mon[91852]: osd.0 marked itself down and dead 2024-04-26T05:02:27.983 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:27 smithi012 ceph-mon[95978]: osd.0 marked itself down and dead 2024-04-26T05:02:27.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:27 smithi116 ceph-mon[85502]: osd.0 marked itself down and dead 2024-04-26T05:02:28.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:02:27 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:02:27] "GET /metrics HTTP/1.1" 200 33837 "" "Prometheus/2.43.0" 2024-04-26T05:02:28.570 INFO:teuthology.orchestra.run.smithi012.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:02:28.583 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:28 smithi012 podman[102728]: 2024-04-26 05:02:28.331161214 +0000 UTC m=+1.777119128 container died 25b6843697db7df36b1e11966b7d52c4bc749615f32e736869bfce9d6cf1ac89 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0, CEPH_POINT_RELEASE=, RELEASE=quincy-c61472c, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-04-26T05:02:28.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:28 smithi012 ceph-mon[91852]: pgmap v102: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:28.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:28 smithi012 ceph-mon[91852]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T05:02:28.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:28 smithi012 ceph-mon[91852]: osdmap e81: 8 total, 7 up, 8 in 2024-04-26T05:02:28.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:28 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:28.927 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:28 smithi012 podman[102728]: 2024-04-26 05:02:28.663317325 +0000 UTC m=+2.109275237 container cleanup 25b6843697db7df36b1e11966b7d52c4bc749615f32e736869bfce9d6cf1ac89 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=quincy-c61472c, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240424) 2024-04-26T05:02:28.927 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:28 smithi012 bash[102728]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0 2024-04-26T05:02:28.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:28 smithi012 ceph-mon[95978]: pgmap v102: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:28.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:28 smithi012 ceph-mon[95978]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T05:02:28.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:28 smithi012 ceph-mon[95978]: osdmap e81: 8 total, 7 up, 8 in 2024-04-26T05:02:28.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:28 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:28.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:28 smithi116 ceph-mon[85502]: pgmap v102: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:28.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:28 smithi116 ceph-mon[85502]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T05:02:28.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:28 smithi116 ceph-mon[85502]: osdmap e81: 8 total, 7 up, 8 in 2024-04-26T05:02:28.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:28 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:28.992 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:02:28 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:02:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi116.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:02:29.205 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'sleep 60' 2024-04-26T05:02:29.482 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:02:29.513 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:29 smithi012 podman[102847]: 2024-04-26 05:02:29.38515599 +0000 UTC m=+1.228394247 container remove 25b6843697db7df36b1e11966b7d52c4bc749615f32e736869bfce9d6cf1ac89 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, ceph=True, RELEASE=quincy-c61472c, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-04-26T05:02:29.864 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:29 smithi012 ceph-mon[91852]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:02:29.865 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:29 smithi012 ceph-mon[91852]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:02:29.865 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:29 smithi012 ceph-mon[91852]: osdmap e82: 8 total, 7 up, 8 in 2024-04-26T05:02:29.865 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:29 smithi012 ceph-mon[95978]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:02:29.865 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:29 smithi012 ceph-mon[95978]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:02:29.865 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:29 smithi012 ceph-mon[95978]: osdmap e82: 8 total, 7 up, 8 in 2024-04-26T05:02:29.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:29 smithi116 ceph-mon[85502]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:02:29.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:29 smithi116 ceph-mon[85502]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:02:29.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:29 smithi116 ceph-mon[85502]: osdmap e82: 8 total, 7 up, 8 in 2024-04-26T05:02:30.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:30 smithi012 ceph-mon[91852]: pgmap v105: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:30.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:30 smithi012 ceph-mon[95978]: pgmap v105: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:30.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:30 smithi116 ceph-mon[85502]: pgmap v105: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:31.577 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:31 smithi012 podman[103107]: 2024-04-26 05:02:31.214192306 +0000 UTC m=+0.018899383 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:02:31.577 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:31 smithi012 podman[103107]: 2024-04-26 05:02:31.32433221 +0000 UTC m=+0.129039317 container create b3dea0a5b00e04e3f70bc918c4c887273cbdf4d4e05017f181f6499ff7752f30 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T05:02:31.577 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:31 smithi012 podman[103107]: 2024-04-26 05:02:31.518559578 +0000 UTC m=+0.323266641 container init b3dea0a5b00e04e3f70bc918c4c887273cbdf4d4e05017f181f6499ff7752f30 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T05:02:31.578 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:31 smithi012 podman[103107]: 2024-04-26 05:02:31.521782586 +0000 UTC m=+0.326489650 container start b3dea0a5b00e04e3f70bc918c4c887273cbdf4d4e05017f181f6499ff7752f30 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:02:31.578 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:31 smithi012 podman[103107]: 2024-04-26 05:02:31.576911639 +0000 UTC m=+0.381618704 container attach b3dea0a5b00e04e3f70bc918c4c887273cbdf4d4e05017f181f6499ff7752f30 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0-deactivate, ceph=True, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-26T05:02:32.114 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:31 smithi012 podman[103107]: 2024-04-26 05:02:31.828226316 +0000 UTC m=+0.632933379 container died b3dea0a5b00e04e3f70bc918c4c887273cbdf4d4e05017f181f6499ff7752f30 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:02:32.645 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:32 smithi012 podman[103133]: 2024-04-26 05:02:32.380666883 +0000 UTC m=+0.547429977 container remove b3dea0a5b00e04e3f70bc918c4c887273cbdf4d4e05017f181f6499ff7752f30 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T05:02:32.645 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:32 smithi012 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@osd.0.service: Deactivated successfully. 2024-04-26T05:02:32.645 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:32 smithi012 systemd[1]: Stopped Ceph osd.0 for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T05:02:32.645 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:32 smithi012 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@osd.0.service: Consumed 5.875s CPU time. 2024-04-26T05:02:32.645 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:32 smithi012 systemd[1]: Starting Ceph osd.0 for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T05:02:32.931 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:32 smithi012 ceph-mon[91852]: pgmap v106: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:32.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:32 smithi012 ceph-mon[95978]: pgmap v106: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:32.932 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:32 smithi012 podman[103239]: 2024-04-26 05:02:32.831672713 +0000 UTC m=+0.105076176 container create abe5059efdb6f9cb5797308386a1bbde6ded824c38ae960fe014c6d815bab1b0 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-04-26T05:02:32.932 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:32 smithi012 podman[103239]: 2024-04-26 05:02:32.745790798 +0000 UTC m=+0.019194274 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:02:32.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:32 smithi116 ceph-mon[85502]: pgmap v106: 73 pgs: 11 stale+active+clean, 62 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:33.263 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:33 smithi012 podman[103239]: 2024-04-26 05:02:33.017149737 +0000 UTC m=+0.290553221 container init abe5059efdb6f9cb5797308386a1bbde6ded824c38ae960fe014c6d815bab1b0 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0-activate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0) 2024-04-26T05:02:33.263 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:33 smithi012 podman[103239]: 2024-04-26 05:02:33.020440598 +0000 UTC m=+0.293844064 container start abe5059efdb6f9cb5797308386a1bbde6ded824c38ae960fe014c6d815bab1b0 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0-activate, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux ) 2024-04-26T05:02:33.263 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:33 smithi012 podman[103239]: 2024-04-26 05:02:33.092349319 +0000 UTC m=+0.365752806 container attach abe5059efdb6f9cb5797308386a1bbde6ded824c38ae960fe014c6d815bab1b0 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0-activate, ceph=True, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240424) 2024-04-26T05:02:33.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:33 smithi012 ceph-mon[91852]: Health check failed: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T05:02:33.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:33 smithi012 ceph-mon[95978]: Health check failed: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T05:02:33.764 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:33 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0-activate[103253]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-26T05:02:33.764 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:33 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0-activate[103253]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-26T05:02:33.764 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:33 smithi012 bash[103239]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-26T05:02:33.764 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:33 smithi012 bash[103239]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-26T05:02:33.764 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:33 smithi012 bash[103239]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-26T05:02:33.764 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:33 smithi012 bash[103239]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-26T05:02:33.764 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:33 smithi012 bash[103239]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-26T05:02:33.764 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:33 smithi012 bash[103239]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-26T05:02:33.764 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:33 smithi012 bash[103239]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-26T05:02:33.764 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:33 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0-activate[103253]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-26T05:02:33.764 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:33 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0-activate[103253]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-26T05:02:33.765 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:33 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0-activate[103253]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-26T05:02:33.765 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:33 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0-activate[103253]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-26T05:02:33.765 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:33 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0-activate[103253]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-26T05:02:33.765 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:33 smithi012 podman[103239]: 2024-04-26 05:02:33.503388519 +0000 UTC m=+0.776791983 container died abe5059efdb6f9cb5797308386a1bbde6ded824c38ae960fe014c6d815bab1b0 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0-activate, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-04-26T05:02:33.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:33 smithi116 ceph-mon[85502]: Health check failed: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T05:02:34.456 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:34 smithi012 podman[103372]: 2024-04-26 05:02:34.12171771 +0000 UTC m=+0.613245551 container remove abe5059efdb6f9cb5797308386a1bbde6ded824c38ae960fe014c6d815bab1b0 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0-activate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-04-26T05:02:34.456 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:34 smithi012 podman[103438]: 2024-04-26 05:02:34.249645885 +0000 UTC m=+0.019387247 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:02:34.456 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:34 smithi012 podman[103438]: 2024-04-26 05:02:34.355817688 +0000 UTC m=+0.125559036 container create 512ac46479c20e01647c0cc6f6ce4f1dd256baf85133d58dc3ec4437d19e9ad4 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-04-26T05:02:34.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:34 smithi012 ceph-mon[91852]: pgmap v107: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-04-26T05:02:34.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:34 smithi012 ceph-mon[95978]: pgmap v107: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-04-26T05:02:34.763 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:34 smithi012 podman[103438]: 2024-04-26 05:02:34.583532848 +0000 UTC m=+0.353274196 container init 512ac46479c20e01647c0cc6f6ce4f1dd256baf85133d58dc3ec4437d19e9ad4 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-04-26T05:02:34.763 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:34 smithi012 podman[103438]: 2024-04-26 05:02:34.587242975 +0000 UTC m=+0.356984329 container start 512ac46479c20e01647c0cc6f6ce4f1dd256baf85133d58dc3ec4437d19e9ad4 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb) 2024-04-26T05:02:34.763 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:34 smithi012 ceph-osd[103457]: -- 172.21.15.12:0/829283208 <== mon.2 v2:172.21.15.12:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x559a2765d680 con 0x559a27627400 2024-04-26T05:02:34.764 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:34 smithi012 bash[103438]: 512ac46479c20e01647c0cc6f6ce4f1dd256baf85133d58dc3ec4437d19e9ad4 2024-04-26T05:02:34.764 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:34 smithi012 systemd[1]: Started Ceph osd.0 for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T05:02:34.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:34 smithi116 ceph-mon[85502]: pgmap v107: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-04-26T05:02:35.513 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:35 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0[103452]: 2024-04-26T05:02:35.125+0000 7fa5950707c0 -1 Falling back to public interface 2024-04-26T05:02:35.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:35 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:35.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:35 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:35.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:35 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:02:36.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:35 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:36.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:35 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:36.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:35 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:02:36.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:35 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:36.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:35 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:36.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:35 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:02:36.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:36 smithi116 ceph-mon[85502]: pgmap v108: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-04-26T05:02:37.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:36 smithi012 ceph-mon[91852]: pgmap v108: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-04-26T05:02:37.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:36 smithi012 ceph-mon[95978]: pgmap v108: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-04-26T05:02:38.207 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:37 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0[103452]: 2024-04-26T05:02:37.863+0000 7fa5950707c0 -1 osd.0 80 log_to_monitors true 2024-04-26T05:02:38.207 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:02:37 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:02:37] "GET /metrics HTTP/1.1" 200 33837 "" "Prometheus/2.43.0" 2024-04-26T05:02:38.991 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:02:38 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:02:38.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi116.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:02:38.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:38 smithi116 ceph-mon[85502]: pgmap v109: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-04-26T05:02:38.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:38 smithi116 ceph-mon[85502]: from='osd.0 [v2:172.21.15.12:6802/1787752712,v1:172.21.15.12:6803/1787752712]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T05:02:38.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:38 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:38.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:38 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:38.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:38 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:38.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:38 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:39.013 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:02:38 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0[103452]: 2024-04-26T05:02:38.759+0000 7fa5864c8700 -1 osd.0 80 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T05:02:39.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:38 smithi012 ceph-mon[91852]: pgmap v109: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-04-26T05:02:39.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:38 smithi012 ceph-mon[91852]: from='osd.0 [v2:172.21.15.12:6802/1787752712,v1:172.21.15.12:6803/1787752712]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T05:02:39.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:38 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:39.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:38 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:39.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:38 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:39.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:38 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:39.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:38 smithi012 ceph-mon[95978]: pgmap v109: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-04-26T05:02:39.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:38 smithi012 ceph-mon[95978]: from='osd.0 [v2:172.21.15.12:6802/1787752712,v1:172.21.15.12:6803/1787752712]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T05:02:39.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:38 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:39.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:38 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:39.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:38 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:39.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:38 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:40.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:39 smithi012 ceph-mon[91852]: from='osd.0 [v2:172.21.15.12:6802/1787752712,v1:172.21.15.12:6803/1787752712]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T05:02:40.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:39 smithi012 ceph-mon[91852]: osdmap e83: 8 total, 7 up, 8 in 2024-04-26T05:02:40.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:39 smithi012 ceph-mon[91852]: from='osd.0 [v2:172.21.15.12:6802/1787752712,v1:172.21.15.12:6803/1787752712]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-04-26T05:02:40.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:39 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:40.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:39 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:40.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:39 smithi012 ceph-mon[95978]: from='osd.0 [v2:172.21.15.12:6802/1787752712,v1:172.21.15.12:6803/1787752712]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T05:02:40.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:39 smithi012 ceph-mon[95978]: osdmap e83: 8 total, 7 up, 8 in 2024-04-26T05:02:40.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:39 smithi012 ceph-mon[95978]: from='osd.0 [v2:172.21.15.12:6802/1787752712,v1:172.21.15.12:6803/1787752712]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-04-26T05:02:40.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:39 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:40.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:39 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:40.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:39 smithi116 ceph-mon[85502]: from='osd.0 [v2:172.21.15.12:6802/1787752712,v1:172.21.15.12:6803/1787752712]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T05:02:40.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:39 smithi116 ceph-mon[85502]: osdmap e83: 8 total, 7 up, 8 in 2024-04-26T05:02:40.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:39 smithi116 ceph-mon[85502]: from='osd.0 [v2:172.21.15.12:6802/1787752712,v1:172.21.15.12:6803/1787752712]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-04-26T05:02:40.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:39 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:40.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:39 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:41.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:40 smithi012 ceph-mon[95978]: pgmap v111: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-04-26T05:02:41.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:40 smithi012 ceph-mon[95978]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T05:02:41.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:40 smithi012 ceph-mon[95978]: osd.0 [v2:172.21.15.12:6802/1787752712,v1:172.21.15.12:6803/1787752712] boot 2024-04-26T05:02:41.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:40 smithi012 ceph-mon[95978]: osdmap e84: 8 total, 8 up, 8 in 2024-04-26T05:02:41.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:40 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T05:02:41.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:40 smithi012 ceph-mon[91852]: pgmap v111: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-04-26T05:02:41.015 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:40 smithi012 ceph-mon[91852]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T05:02:41.015 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:40 smithi012 ceph-mon[91852]: osd.0 [v2:172.21.15.12:6802/1787752712,v1:172.21.15.12:6803/1787752712] boot 2024-04-26T05:02:41.015 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:40 smithi012 ceph-mon[91852]: osdmap e84: 8 total, 8 up, 8 in 2024-04-26T05:02:41.015 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:40 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T05:02:41.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:40 smithi116 ceph-mon[85502]: pgmap v111: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-04-26T05:02:41.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:40 smithi116 ceph-mon[85502]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T05:02:41.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:40 smithi116 ceph-mon[85502]: osd.0 [v2:172.21.15.12:6802/1787752712,v1:172.21.15.12:6803/1787752712] boot 2024-04-26T05:02:41.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:40 smithi116 ceph-mon[85502]: osdmap e84: 8 total, 8 up, 8 in 2024-04-26T05:02:41.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:40 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T05:02:42.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:41 smithi012 ceph-mon[91852]: osdmap e85: 8 total, 8 up, 8 in 2024-04-26T05:02:42.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:41 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:42.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:41 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:42.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:41 smithi012 ceph-mon[95978]: osdmap e85: 8 total, 8 up, 8 in 2024-04-26T05:02:42.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:41 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:42.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:41 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:42.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:41 smithi116 ceph-mon[85502]: osdmap e85: 8 total, 8 up, 8 in 2024-04-26T05:02:42.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:41 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:42.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:41 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:42.897 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:42 smithi012 ceph-mon[91852]: pgmap v114: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-04-26T05:02:42.898 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:42 smithi012 ceph-mon[95978]: pgmap v114: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-04-26T05:02:43.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:42 smithi116 ceph-mon[85502]: pgmap v114: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2024-04-26T05:02:44.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:44 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:44.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:44 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:44.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:44 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:44.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:44 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:02:44.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:44 smithi116 ceph-mon[85502]: pgmap v115: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T05:02:44.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:44 smithi116 ceph-mon[85502]: pgmap v116: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T05:02:44.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:44 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:44.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:44 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:02:44.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:44 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:44.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:44 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:02:44.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:44 smithi116 ceph-mon[85502]: pgmap v117: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T05:02:44.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:44 smithi116 ceph-mon[85502]: pgmap v118: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T05:02:44.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:44 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:44.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:44 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:44.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:44 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:44.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:44 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:44.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:44 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:02:44.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:44 smithi012 ceph-mon[91852]: pgmap v115: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T05:02:44.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:44 smithi012 ceph-mon[91852]: pgmap v116: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T05:02:44.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:44 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:44.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:44 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:02:44.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:44 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:44.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:44 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:02:44.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:44 smithi012 ceph-mon[91852]: pgmap v117: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T05:02:44.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:44 smithi012 ceph-mon[91852]: pgmap v118: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T05:02:44.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:44 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:44.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:44 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:44.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:44 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:44.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:44 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:44.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:44 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:02:44.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:44 smithi012 ceph-mon[95978]: pgmap v115: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T05:02:44.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:44 smithi012 ceph-mon[95978]: pgmap v116: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T05:02:44.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:44 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:44.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:44 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:02:44.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:44 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:44.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:44 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:02:44.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:44 smithi012 ceph-mon[95978]: pgmap v117: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T05:02:44.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:44 smithi012 ceph-mon[95978]: pgmap v118: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T05:02:44.515 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:44 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:45.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:45 smithi116 ceph-mon[85502]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T05:02:45.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:45 smithi012 ceph-mon[91852]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T05:02:45.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:45 smithi012 ceph-mon[95978]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T05:02:46.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:46 smithi012 ceph-mon[91852]: pgmap v119: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:46.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:46 smithi012 ceph-mon[95978]: pgmap v119: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:46.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:46 smithi116 ceph-mon[85502]: pgmap v119: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:47.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:47 smithi012 ceph-mon[91852]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded) 2024-04-26T05:02:47.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:47 smithi012 ceph-mon[95978]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded) 2024-04-26T05:02:47.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:47 smithi116 ceph-mon[85502]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded) 2024-04-26T05:02:48.250 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:02:47 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:02:47] "GET /metrics HTTP/1.1" 200 33746 "" "Prometheus/2.43.0" 2024-04-26T05:02:48.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:48 smithi012 ceph-mon[91852]: pgmap v120: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:48.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:48 smithi012 ceph-mon[95978]: pgmap v120: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:48.660 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:48 smithi116 ceph-mon[85502]: pgmap v120: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:48.991 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:02:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:02:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi116.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:02:50.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:50 smithi116 ceph-mon[85502]: pgmap v121: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:50.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:50 smithi012 ceph-mon[91852]: pgmap v121: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:50.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:50 smithi012 ceph-mon[95978]: pgmap v121: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:52.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:52 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:52.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:52 smithi116 ceph-mon[85502]: pgmap v122: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:52.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:52 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:52.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:52 smithi012 ceph-mon[91852]: pgmap v122: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:52.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:52 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:52.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:52 smithi012 ceph-mon[95978]: pgmap v122: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:54.747 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:54 smithi116 ceph-mon[85502]: pgmap v123: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:54.768 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:54 smithi012 ceph-mon[91852]: pgmap v123: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:54.768 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:54 smithi012 ceph-mon[95978]: pgmap v123: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:56.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:56 smithi116 ceph-mon[85502]: pgmap v124: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:56.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:56 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:56.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:56 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:56.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:56 smithi012 ceph-mon[91852]: pgmap v124: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:56.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:56 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:56.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:56 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:56.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:56 smithi012 ceph-mon[95978]: pgmap v124: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:56.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:56 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:02:56.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:56 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:58.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:02:57 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:02:57] "GET /metrics HTTP/1.1" 200 33895 "" "Prometheus/2.43.0" 2024-04-26T05:02:58.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:02:58 smithi116 ceph-mon[85502]: pgmap v125: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:58.742 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:02:58 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:02:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi012:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.12\", device_class=\"ssd\", hostname=\"smithi012\", instance=\"smithi116.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.12\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T05:02:58.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:02:58 smithi012 ceph-mon[91852]: pgmap v125: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:58.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:02:58 smithi012 ceph-mon[95978]: pgmap v125: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:00.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:00 smithi116 ceph-mon[85502]: pgmap v126: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:00.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:00 smithi012 ceph-mon[91852]: pgmap v126: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:00.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:00 smithi012 ceph-mon[95978]: pgmap v126: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:02.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:02 smithi116 ceph-mon[85502]: pgmap v127: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:02.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:02 smithi012 ceph-mon[91852]: pgmap v127: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:02.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:02 smithi012 ceph-mon[95978]: pgmap v127: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:04.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:04 smithi116 ceph-mon[85502]: pgmap v128: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:04.764 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:04 smithi012 ceph-mon[91852]: pgmap v128: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:04.764 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:04 smithi012 ceph-mon[95978]: pgmap v128: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:06.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:06 smithi116 ceph-mon[85502]: pgmap v129: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:06.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:06 smithi012 ceph-mon[91852]: pgmap v129: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:06.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:06 smithi012 ceph-mon[95978]: pgmap v129: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:08.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:03:07 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:03:07] "GET /metrics HTTP/1.1" 200 33895 "" "Prometheus/2.43.0" 2024-04-26T05:03:08.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:08 smithi116 ceph-mon[85502]: pgmap v130: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:08.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:08 smithi012 ceph-mon[91852]: pgmap v130: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:08.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:08 smithi012 ceph-mon[95978]: pgmap v130: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:10.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:10 smithi116 ceph-mon[85502]: pgmap v131: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:10.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:10 smithi012 ceph-mon[91852]: pgmap v131: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:10.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:10 smithi012 ceph-mon[95978]: pgmap v131: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:11.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:11 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:11.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:11 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:11.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:11 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:12.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:12 smithi116 ceph-mon[85502]: pgmap v132: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:12.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:12 smithi012 ceph-mon[91852]: pgmap v132: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:12.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:12 smithi012 ceph-mon[95978]: pgmap v132: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:14.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:14 smithi116 ceph-mon[85502]: pgmap v133: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:14.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:14 smithi012 ceph-mon[91852]: pgmap v133: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:14.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:14 smithi012 ceph-mon[95978]: pgmap v133: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:16.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:16 smithi116 ceph-mon[85502]: pgmap v134: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:16.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:16 smithi012 ceph-mon[91852]: pgmap v134: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:16.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:16 smithi012 ceph-mon[95978]: pgmap v134: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:18.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:03:17 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:03:17] "GET /metrics HTTP/1.1" 200 33898 "" "Prometheus/2.43.0" 2024-04-26T05:03:18.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:18 smithi116 ceph-mon[85502]: pgmap v135: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:18.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:18 smithi012 ceph-mon[91852]: pgmap v135: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:18.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:18 smithi012 ceph-mon[95978]: pgmap v135: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:20.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:20 smithi012 ceph-mon[91852]: pgmap v136: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:20.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:20 smithi012 ceph-mon[95978]: pgmap v136: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:20.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:20 smithi116 ceph-mon[85502]: pgmap v136: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:22.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:22 smithi012 ceph-mon[91852]: pgmap v137: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:22.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:22 smithi012 ceph-mon[95978]: pgmap v137: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:22.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:22 smithi116 ceph-mon[85502]: pgmap v137: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:24.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:24 smithi012 ceph-mon[91852]: pgmap v138: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:24.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:24 smithi012 ceph-mon[95978]: pgmap v138: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:24.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:24 smithi116 ceph-mon[85502]: pgmap v138: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:26.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:26 smithi012 ceph-mon[91852]: pgmap v139: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:26.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:26 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:26.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:26 smithi012 ceph-mon[95978]: pgmap v139: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:26.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:26 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:26.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:26 smithi116 ceph-mon[85502]: pgmap v139: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:26.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:26 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:28.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:03:27 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:03:27] "GET /metrics HTTP/1.1" 200 33900 "" "Prometheus/2.43.0" 2024-04-26T05:03:28.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:28 smithi012 ceph-mon[91852]: pgmap v140: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:28.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:28 smithi012 ceph-mon[95978]: pgmap v140: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:28.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:28 smithi116 ceph-mon[85502]: pgmap v140: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:30.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:30 smithi012 ceph-mon[91852]: pgmap v141: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:30.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:30 smithi012 ceph-mon[95978]: pgmap v141: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:30.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:30 smithi116 ceph-mon[85502]: pgmap v141: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:31.627 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade resume' 2024-04-26T05:03:31.898 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:03:32.612 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:32 smithi012 ceph-mon[91852]: pgmap v142: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:32.613 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:32 smithi012 ceph-mon[95978]: pgmap v142: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:32.793 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:32 smithi116 ceph-mon[85502]: pgmap v142: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:33.595 INFO:teuthology.orchestra.run.smithi012.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:03:34.154 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'echo wait for half of osds to upgrade' 2024-04-26T05:03:34.682 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:34 smithi012 ceph-mon[91852]: pgmap v143: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:34.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:34 smithi012 ceph-mon[91852]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:03:34.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:34 smithi012 ceph-mon[91852]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:03:34.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:34 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:34.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:34 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:03:34.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:34 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:03:34.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:34 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:03:34.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:34 smithi012 ceph-mon[91852]: pgmap v144: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:34.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:34 smithi012 ceph-mon[91852]: pgmap v145: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:34.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:34 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:34.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:34 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:03:34.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:34 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:03:34.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:34 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:03:34.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:34 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:03:34.683 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:34 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T05:03:34.684 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:34 smithi012 ceph-mon[91852]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T05:03:34.684 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:34 smithi012 ceph-mon[91852]: Upgrade: osd.1 is safe to restart 2024-04-26T05:03:34.684 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:34 smithi012 ceph-mon[95978]: pgmap v143: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:34.684 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:34 smithi012 ceph-mon[95978]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:03:34.684 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:34 smithi012 ceph-mon[95978]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:03:34.684 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:34 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:34.684 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:34 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:03:34.684 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:34 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:03:34.684 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:34 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:03:34.684 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:34 smithi012 ceph-mon[95978]: pgmap v144: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:34.684 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:34 smithi012 ceph-mon[95978]: pgmap v145: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:34.684 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:34 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:34.684 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:34 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:03:34.684 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:34 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:03:34.685 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:34 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:03:34.685 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:34 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:03:34.685 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:34 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T05:03:34.685 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:34 smithi012 ceph-mon[95978]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T05:03:34.685 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:34 smithi012 ceph-mon[95978]: Upgrade: osd.1 is safe to restart 2024-04-26T05:03:34.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:34 smithi116 ceph-mon[85502]: pgmap v143: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:34.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:34 smithi116 ceph-mon[85502]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:03:34.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:34 smithi116 ceph-mon[85502]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:03:34.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:34 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:34.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:34 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:03:34.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:34 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:03:34.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:34 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:03:34.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:34 smithi116 ceph-mon[85502]: pgmap v144: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:34.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:34 smithi116 ceph-mon[85502]: pgmap v145: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:34.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:34 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:34.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:34 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:03:34.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:34 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:03:34.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:34 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:03:34.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:34 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:03:34.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:34 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T05:03:34.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:34 smithi116 ceph-mon[85502]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T05:03:34.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:34 smithi116 ceph-mon[85502]: Upgrade: osd.1 is safe to restart 2024-04-26T05:03:35.058 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:03:36.898 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:36 smithi012 ceph-mon[91852]: Upgrade: Updating osd.1 2024-04-26T05:03:36.898 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:36 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:36.898 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:36 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T05:03:36.898 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:36 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:03:36.898 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:36 smithi012 ceph-mon[91852]: Deploying daemon osd.1 on smithi012 2024-04-26T05:03:36.898 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:36 smithi012 ceph-mon[91852]: pgmap v146: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:36.899 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:36 smithi012 ceph-mon[95978]: Upgrade: Updating osd.1 2024-04-26T05:03:36.899 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:36 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:36.899 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:36 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T05:03:36.899 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:36 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:03:36.899 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:36 smithi012 ceph-mon[95978]: Deploying daemon osd.1 on smithi012 2024-04-26T05:03:36.899 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:36 smithi012 ceph-mon[95978]: pgmap v146: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:36.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:36 smithi116 ceph-mon[85502]: Upgrade: Updating osd.1 2024-04-26T05:03:36.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:36 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:36.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:36 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T05:03:36.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:36 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:03:36.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:36 smithi116 ceph-mon[85502]: Deploying daemon osd.1 on smithi012 2024-04-26T05:03:36.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:36 smithi116 ceph-mon[85502]: pgmap v146: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:38.250 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:03:37 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:03:37] "GET /metrics HTTP/1.1" 200 33900 "" "Prometheus/2.43.0" 2024-04-26T05:03:38.550 INFO:teuthology.orchestra.run.smithi012.stdout:wait for half of osds to upgrade 2024-04-26T05:03:38.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:38 smithi116 ceph-mon[85502]: pgmap v147: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:39.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:38 smithi012 ceph-mon[91852]: pgmap v147: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:39.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:38 smithi012 ceph-mon[95978]: pgmap v147: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:39.978 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2024-04-26T05:03:40.214 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:39 smithi012 systemd[1]: Stopping Ceph osd.1 for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T05:03:40.284 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:03:40.513 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:40 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1[47231]: 2024-04-26T05:03:40.212+0000 7fb5a9796700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:03:40.513 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:40 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1[47231]: 2024-04-26T05:03:40.212+0000 7fb5a9796700 -1 osd.1 85 *** Got signal Terminated *** 2024-04-26T05:03:40.513 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:40 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1[47231]: 2024-04-26T05:03:40.212+0000 7fb5a9796700 -1 osd.1 85 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:03:40.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:40 smithi116 ceph-mon[85502]: pgmap v148: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:40.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:40 smithi116 ceph-mon[85502]: osd.1 marked itself down and dead 2024-04-26T05:03:41.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:40 smithi012 ceph-mon[91852]: pgmap v148: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:41.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:40 smithi012 ceph-mon[91852]: osd.1 marked itself down and dead 2024-04-26T05:03:41.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:40 smithi012 ceph-mon[95978]: pgmap v148: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:41.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:40 smithi012 ceph-mon[95978]: osd.1 marked itself down and dead 2024-04-26T05:03:41.941 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:41 smithi012 ceph-mon[91852]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T05:03:41.941 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:41 smithi012 ceph-mon[91852]: osdmap e86: 8 total, 7 up, 8 in 2024-04-26T05:03:41.941 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:41 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:41.941 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:41 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:41.941 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:41 smithi012 ceph-mon[95978]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T05:03:41.942 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:41 smithi012 ceph-mon[95978]: osdmap e86: 8 total, 7 up, 8 in 2024-04-26T05:03:41.942 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:41 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:41.942 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:41 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:41.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:41 smithi116 ceph-mon[85502]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T05:03:41.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:41 smithi116 ceph-mon[85502]: osdmap e86: 8 total, 7 up, 8 in 2024-04-26T05:03:41.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:41 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:41.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:41 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:42.263 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:41 smithi012 podman[108346]: 2024-04-26 05:03:41.940264832 +0000 UTC m=+1.841387651 container died 8117a7da3ae3baff8c44de045a31e6e7de9c2e02f95770004975fc065d48f7ed (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=quincy-c61472c, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:03:42.704 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:42 smithi012 podman[108346]: 2024-04-26 05:03:42.33310708 +0000 UTC m=+2.234229875 container cleanup 8117a7da3ae3baff8c44de045a31e6e7de9c2e02f95770004975fc065d48f7ed (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=quincy-c61472c, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-26T05:03:42.705 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:42 smithi012 bash[108346]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1 2024-04-26T05:03:42.705 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:42 smithi012 podman[108440]: 2024-04-26 05:03:42.484467918 +0000 UTC m=+1.369469211 container remove 8117a7da3ae3baff8c44de045a31e6e7de9c2e02f95770004975fc065d48f7ed (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-c61472c, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-04-26T05:03:42.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:42 smithi116 ceph-mon[85502]: pgmap v150: 73 pgs: 9 stale+active+clean, 64 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:42.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:42 smithi116 ceph-mon[85502]: osdmap e87: 8 total, 7 up, 8 in 2024-04-26T05:03:42.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:42 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/3875790404' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:03:42.996 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:42 smithi012 ceph-mon[91852]: pgmap v150: 73 pgs: 9 stale+active+clean, 64 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:42.996 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:42 smithi012 ceph-mon[91852]: osdmap e87: 8 total, 7 up, 8 in 2024-04-26T05:03:42.996 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:42 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/3875790404' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:03:42.997 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:42 smithi012 ceph-mon[95978]: pgmap v150: 73 pgs: 9 stale+active+clean, 64 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:42.997 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:42 smithi012 ceph-mon[95978]: osdmap e87: 8 total, 7 up, 8 in 2024-04-26T05:03:42.997 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:42 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/3875790404' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:03:42.997 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:42 smithi012 podman[108585]: 2024-04-26 05:03:42.902164321 +0000 UTC m=+0.114406248 container create 12ac7925af635d7b9d4f171ebf1129531affe460c280654d9484e22d56aaeade (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T05:03:42.997 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:42 smithi012 podman[108585]: 2024-04-26 05:03:42.806366172 +0000 UTC m=+0.018608063 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:03:43.263 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:43 smithi012 podman[108585]: 2024-04-26 05:03:43.104503096 +0000 UTC m=+0.316744978 container init 12ac7925af635d7b9d4f171ebf1129531affe460c280654d9484e22d56aaeade (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1-deactivate, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-04-26T05:03:43.263 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:43 smithi012 podman[108585]: 2024-04-26 05:03:43.108391 +0000 UTC m=+0.320632888 container start 12ac7925af635d7b9d4f171ebf1129531affe460c280654d9484e22d56aaeade (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-04-26T05:03:43.263 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:43 smithi012 podman[108585]: 2024-04-26 05:03:43.179553901 +0000 UTC m=+0.391795798 container attach 12ac7925af635d7b9d4f171ebf1129531affe460c280654d9484e22d56aaeade (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1-deactivate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, GIT_CLEAN=True) 2024-04-26T05:03:43.725 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:43 smithi012 podman[108585]: 2024-04-26 05:03:43.414738889 +0000 UTC m=+0.626980773 container died 12ac7925af635d7b9d4f171ebf1129531affe460c280654d9484e22d56aaeade (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_BRANCH=HEAD) 2024-04-26T05:03:44.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:43 smithi012 ceph-mon[91852]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T05:03:44.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:43 smithi012 ceph-mon[95978]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T05:03:44.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:43 smithi116 ceph-mon[85502]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T05:03:44.477 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:44 smithi012 podman[108611]: 2024-04-26 05:03:44.017145891 +0000 UTC m=+0.596930970 container remove 12ac7925af635d7b9d4f171ebf1129531affe460c280654d9484e22d56aaeade (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T05:03:44.478 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:44 smithi012 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@osd.1.service: Deactivated successfully. 2024-04-26T05:03:44.478 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:44 smithi012 systemd[1]: Stopped Ceph osd.1 for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T05:03:44.478 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:44 smithi012 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@osd.1.service: Consumed 6.239s CPU time. 2024-04-26T05:03:44.478 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:44 smithi012 systemd[1]: Starting Ceph osd.1 for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T05:03:44.760 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:44 smithi012 podman[108736]: 2024-04-26 05:03:44.476546703 +0000 UTC m=+0.107932974 container create 273da712d5aea2f3f7213d0e9ddae528d162f86c6032ebcbfec39dceae454979 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1-activate, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb) 2024-04-26T05:03:44.760 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:44 smithi012 podman[108736]: 2024-04-26 05:03:44.39170865 +0000 UTC m=+0.023094937 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:03:44.760 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:44 smithi012 podman[108736]: 2024-04-26 05:03:44.687731761 +0000 UTC m=+0.319118024 container init 273da712d5aea2f3f7213d0e9ddae528d162f86c6032ebcbfec39dceae454979 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1-activate, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-04-26T05:03:44.760 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:44 smithi012 podman[108736]: 2024-04-26 05:03:44.690904419 +0000 UTC m=+0.322290689 container start 273da712d5aea2f3f7213d0e9ddae528d162f86c6032ebcbfec39dceae454979 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1-activate, RELEASE=squid-b22e2eb, GIT_CLEAN=True, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-04-26T05:03:45.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:44 smithi012 ceph-mon[91852]: pgmap v152: 73 pgs: 2 active+undersized+degraded, 1 stale+active+clean, 26 active+undersized, 44 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-26T05:03:45.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:44 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/2523479525' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:03:45.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:44 smithi012 ceph-mon[95978]: pgmap v152: 73 pgs: 2 active+undersized+degraded, 1 stale+active+clean, 26 active+undersized, 44 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-26T05:03:45.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:44 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/2523479525' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:03:45.014 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:44 smithi012 podman[108736]: 2024-04-26 05:03:44.762597502 +0000 UTC m=+0.393983768 container attach 273da712d5aea2f3f7213d0e9ddae528d162f86c6032ebcbfec39dceae454979 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1-activate, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-04-26T05:03:45.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:44 smithi116 ceph-mon[85502]: pgmap v152: 73 pgs: 2 active+undersized+degraded, 1 stale+active+clean, 26 active+undersized, 44 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-26T05:03:45.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:44 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/2523479525' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:03:45.475 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:45 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1-activate[108754]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-26T05:03:45.476 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:45 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1-activate[108754]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-26T05:03:45.476 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:45 smithi012 bash[108736]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-26T05:03:45.476 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:45 smithi012 bash[108736]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-26T05:03:45.476 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:45 smithi012 bash[108736]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-26T05:03:45.476 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:45 smithi012 bash[108736]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-26T05:03:45.476 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:45 smithi012 bash[108736]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-26T05:03:45.476 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:45 smithi012 bash[108736]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-26T05:03:45.476 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:45 smithi012 bash[108736]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-26T05:03:45.477 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:45 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1-activate[108754]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-26T05:03:45.477 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:45 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1-activate[108754]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-26T05:03:45.477 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:45 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1-activate[108754]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-26T05:03:45.477 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:45 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1-activate[108754]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-26T05:03:45.477 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:45 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1-activate[108754]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-26T05:03:45.477 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:45 smithi012 podman[108736]: 2024-04-26 05:03:45.178932832 +0000 UTC m=+0.810319107 container died 273da712d5aea2f3f7213d0e9ddae528d162f86c6032ebcbfec39dceae454979 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1-activate, maintainer=Guillaume Abrioux , ceph=True, RELEASE=squid-b22e2eb, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=) 2024-04-26T05:03:46.083 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:45 smithi012 podman[108876]: 2024-04-26 05:03:45.783988183 +0000 UTC m=+0.599677978 container remove 273da712d5aea2f3f7213d0e9ddae528d162f86c6032ebcbfec39dceae454979 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1-activate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, maintainer=Guillaume Abrioux ) 2024-04-26T05:03:46.083 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:45 smithi012 podman[108938]: 2024-04-26 05:03:45.902197146 +0000 UTC m=+0.018749615 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:03:46.083 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:46 smithi012 podman[108938]: 2024-04-26 05:03:46.001706074 +0000 UTC m=+0.118258514 container create 793612ebfa6621498967a884bf8911571e01d2817efd5da4928e13329b033ade (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-04-26T05:03:46.396 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:46 smithi012 podman[108938]: 2024-04-26 05:03:46.295515663 +0000 UTC m=+0.412068105 container init 793612ebfa6621498967a884bf8911571e01d2817efd5da4928e13329b033ade (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-04-26T05:03:46.396 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:46 smithi012 podman[108938]: 2024-04-26 05:03:46.298694742 +0000 UTC m=+0.415247191 container start 793612ebfa6621498967a884bf8911571e01d2817efd5da4928e13329b033ade (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD) 2024-04-26T05:03:46.396 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:46 smithi012 ceph-osd[108958]: -- 172.21.15.12:0/3366238014 <== mon.2 v2:172.21.15.12:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55c52d785680 con 0x55c52d74ec00 2024-04-26T05:03:46.762 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:46 smithi012 bash[108938]: 793612ebfa6621498967a884bf8911571e01d2817efd5da4928e13329b033ade 2024-04-26T05:03:46.762 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:46 smithi012 systemd[1]: Started Ceph osd.1 for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T05:03:47.013 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:46 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1[108954]: 2024-04-26T05:03:46.853+0000 7f8c769e97c0 -1 Falling back to public interface 2024-04-26T05:03:47.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:46 smithi012 ceph-mon[91852]: pgmap v153: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T05:03:47.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:46 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:47.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:46 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:47.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:46 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:03:47.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:46 smithi012 ceph-mon[95978]: pgmap v153: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T05:03:47.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:46 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:47.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:46 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:47.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:46 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:03:47.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:46 smithi116 ceph-mon[85502]: pgmap v153: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T05:03:47.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:46 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:47.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:46 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:47.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:46 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:03:47.991 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:47 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/3117813202' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:03:47.991 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:47 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/3117813202' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:03:48.064 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:47 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/3117813202' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:03:48.262 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:03:47 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:03:47] "GET /metrics HTTP/1.1" 200 33963 "" "Prometheus/2.43.0" 2024-04-26T05:03:49.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:48 smithi012 ceph-mon[91852]: pgmap v154: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T05:03:49.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:48 smithi012 ceph-mon[95978]: pgmap v154: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T05:03:49.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:48 smithi116 ceph-mon[85502]: pgmap v154: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T05:03:49.763 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:49 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1[108954]: 2024-04-26T05:03:49.579+0000 7f8c769e97c0 -1 osd.1 85 log_to_monitors true 2024-04-26T05:03:49.957 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:49 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/3574783826' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:03:49.957 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:49 smithi116 ceph-mon[85502]: from='osd.1 [v2:172.21.15.12:6810/990497535,v1:172.21.15.12:6811/990497535]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T05:03:50.072 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:49 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/3574783826' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:03:50.072 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:49 smithi012 ceph-mon[91852]: from='osd.1 [v2:172.21.15.12:6810/990497535,v1:172.21.15.12:6811/990497535]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T05:03:50.072 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:49 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/3574783826' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:03:50.072 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:49 smithi012 ceph-mon[95978]: from='osd.1 [v2:172.21.15.12:6810/990497535,v1:172.21.15.12:6811/990497535]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T05:03:50.073 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:03:49 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1[108954]: 2024-04-26T05:03:49.798+0000 7f8c67e41700 -1 osd.1 85 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T05:03:51.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:50 smithi116 ceph-mon[85502]: pgmap v155: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T05:03:51.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:50 smithi116 ceph-mon[85502]: from='osd.1 [v2:172.21.15.12:6810/990497535,v1:172.21.15.12:6811/990497535]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T05:03:51.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:50 smithi116 ceph-mon[85502]: osdmap e88: 8 total, 7 up, 8 in 2024-04-26T05:03:51.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:50 smithi116 ceph-mon[85502]: from='osd.1 [v2:172.21.15.12:6810/990497535,v1:172.21.15.12:6811/990497535]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-04-26T05:03:51.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:50 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:51.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:50 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:51.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:50 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:51.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:50 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:51.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:50 smithi012 ceph-mon[91852]: pgmap v155: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T05:03:51.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:50 smithi012 ceph-mon[91852]: from='osd.1 [v2:172.21.15.12:6810/990497535,v1:172.21.15.12:6811/990497535]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T05:03:51.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:50 smithi012 ceph-mon[91852]: osdmap e88: 8 total, 7 up, 8 in 2024-04-26T05:03:51.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:50 smithi012 ceph-mon[91852]: from='osd.1 [v2:172.21.15.12:6810/990497535,v1:172.21.15.12:6811/990497535]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-04-26T05:03:51.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:50 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:51.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:50 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:51.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:50 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:51.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:50 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:51.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:50 smithi012 ceph-mon[95978]: pgmap v155: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T05:03:51.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:50 smithi012 ceph-mon[95978]: from='osd.1 [v2:172.21.15.12:6810/990497535,v1:172.21.15.12:6811/990497535]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T05:03:51.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:50 smithi012 ceph-mon[95978]: osdmap e88: 8 total, 7 up, 8 in 2024-04-26T05:03:51.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:50 smithi012 ceph-mon[95978]: from='osd.1 [v2:172.21.15.12:6810/990497535,v1:172.21.15.12:6811/990497535]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-04-26T05:03:51.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:50 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:51.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:50 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:51.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:50 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:51.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:50 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:52.211 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:51 smithi012 ceph-mon[95978]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T05:03:52.211 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:51 smithi012 ceph-mon[95978]: osd.1 [v2:172.21.15.12:6810/990497535,v1:172.21.15.12:6811/990497535] boot 2024-04-26T05:03:52.211 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:51 smithi012 ceph-mon[95978]: osdmap e89: 8 total, 8 up, 8 in 2024-04-26T05:03:52.212 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:51 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T05:03:52.212 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:51 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/644831260' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:03:52.212 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:51 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:52.212 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:51 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:52.212 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:51 smithi012 ceph-mon[95978]: osdmap e90: 8 total, 8 up, 8 in 2024-04-26T05:03:52.212 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:51 smithi012 ceph-mon[91852]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T05:03:52.212 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:51 smithi012 ceph-mon[91852]: osd.1 [v2:172.21.15.12:6810/990497535,v1:172.21.15.12:6811/990497535] boot 2024-04-26T05:03:52.212 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:51 smithi012 ceph-mon[91852]: osdmap e89: 8 total, 8 up, 8 in 2024-04-26T05:03:52.212 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:51 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T05:03:52.212 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:51 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/644831260' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:03:52.212 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:51 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:52.212 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:51 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:52.213 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:51 smithi012 ceph-mon[91852]: osdmap e90: 8 total, 8 up, 8 in 2024-04-26T05:03:52.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:51 smithi116 ceph-mon[85502]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T05:03:52.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:51 smithi116 ceph-mon[85502]: osd.1 [v2:172.21.15.12:6810/990497535,v1:172.21.15.12:6811/990497535] boot 2024-04-26T05:03:52.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:51 smithi116 ceph-mon[85502]: osdmap e89: 8 total, 8 up, 8 in 2024-04-26T05:03:52.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:51 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T05:03:52.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:51 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/644831260' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:03:52.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:51 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:52.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:51 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:52.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:51 smithi116 ceph-mon[85502]: osdmap e90: 8 total, 8 up, 8 in 2024-04-26T05:03:53.211 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:52 smithi012 ceph-mon[91852]: pgmap v158: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T05:03:53.211 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:52 smithi012 ceph-mon[91852]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded (PG_DEGRADED) 2024-04-26T05:03:53.212 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:52 smithi012 ceph-mon[95978]: pgmap v158: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T05:03:53.212 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:52 smithi012 ceph-mon[95978]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded (PG_DEGRADED) 2024-04-26T05:03:53.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:52 smithi116 ceph-mon[85502]: pgmap v158: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T05:03:53.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:52 smithi116 ceph-mon[85502]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded (PG_DEGRADED) 2024-04-26T05:03:54.953 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:54 smithi012 ceph-mon[91852]: pgmap v160: 73 pgs: 1 active+undersized+degraded, 11 active+undersized, 61 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2024-04-26T05:03:54.953 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:54 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/787214062' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:03:54.953 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:54 smithi012 ceph-mon[95978]: pgmap v160: 73 pgs: 1 active+undersized+degraded, 11 active+undersized, 61 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2024-04-26T05:03:54.953 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:54 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/787214062' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:03:55.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:54 smithi116 ceph-mon[85502]: pgmap v160: 73 pgs: 1 active+undersized+degraded, 11 active+undersized, 61 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2024-04-26T05:03:55.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:54 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/787214062' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:03:56.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:55 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:56.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:55 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:56.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:55 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:03:56.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:55 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:03:56.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:55 smithi012 ceph-mon[91852]: pgmap v161: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:56.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:55 smithi012 ceph-mon[91852]: pgmap v162: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:56.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:55 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:56.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:55 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:03:56.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:55 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:03:56.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:55 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:03:56.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:55 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:03:56.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:55 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T05:03:56.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:55 smithi012 ceph-mon[91852]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T05:03:56.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:55 smithi012 ceph-mon[91852]: Upgrade: osd.2 is safe to restart 2024-04-26T05:03:56.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:55 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:56.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:55 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:56.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:55 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:03:56.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:55 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:03:56.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:55 smithi012 ceph-mon[95978]: pgmap v161: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:56.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:55 smithi012 ceph-mon[95978]: pgmap v162: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:56.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:55 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:56.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:55 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:03:56.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:55 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:03:56.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:55 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:03:56.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:55 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:03:56.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:55 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T05:03:56.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:55 smithi012 ceph-mon[95978]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T05:03:56.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:55 smithi012 ceph-mon[95978]: Upgrade: osd.2 is safe to restart 2024-04-26T05:03:56.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:55 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:56.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:55 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:56.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:55 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:03:56.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:55 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:03:56.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:55 smithi116 ceph-mon[85502]: pgmap v161: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:56.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:55 smithi116 ceph-mon[85502]: pgmap v162: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:56.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:55 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:56.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:55 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:03:56.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:55 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:03:56.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:55 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:03:56.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:55 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:03:56.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:55 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T05:03:56.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:55 smithi116 ceph-mon[85502]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T05:03:56.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:55 smithi116 ceph-mon[85502]: Upgrade: osd.2 is safe to restart 2024-04-26T05:03:57.258 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:56 smithi012 ceph-mon[91852]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded) 2024-04-26T05:03:57.258 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:56 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:57.258 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:56 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:57.259 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:56 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/1593349748' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:03:57.259 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:56 smithi012 ceph-mon[91852]: Upgrade: Updating osd.2 2024-04-26T05:03:57.259 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:56 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:57.259 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:56 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T05:03:57.259 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:56 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:03:57.259 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:56 smithi012 ceph-mon[91852]: Deploying daemon osd.2 on smithi012 2024-04-26T05:03:57.259 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:56 smithi012 ceph-mon[95978]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded) 2024-04-26T05:03:57.259 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:56 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:57.259 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:56 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:57.259 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:56 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/1593349748' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:03:57.259 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:56 smithi012 ceph-mon[95978]: Upgrade: Updating osd.2 2024-04-26T05:03:57.259 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:56 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:57.259 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:56 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T05:03:57.260 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:56 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:03:57.260 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:56 smithi012 ceph-mon[95978]: Deploying daemon osd.2 on smithi012 2024-04-26T05:03:57.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:56 smithi116 ceph-mon[85502]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded) 2024-04-26T05:03:57.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:56 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:57.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:56 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:57.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:56 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/1593349748' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:03:57.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:56 smithi116 ceph-mon[85502]: Upgrade: Updating osd.2 2024-04-26T05:03:57.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:56 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:03:57.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:56 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T05:03:57.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:56 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:03:57.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:56 smithi116 ceph-mon[85502]: Deploying daemon osd.2 on smithi012 2024-04-26T05:03:58.210 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:57 smithi012 ceph-mon[91852]: pgmap v163: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:58.210 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:58 smithi012 ceph-mon[95978]: pgmap v163: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:58.210 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:03:57 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:03:57] "GET /metrics HTTP/1.1" 200 33960 "" "Prometheus/2.43.0" 2024-04-26T05:03:58.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:57 smithi116 ceph-mon[85502]: pgmap v163: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:59.054 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:03:59 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/1445596395' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:03:59.054 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:03:59 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/1445596395' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:03:59.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:03:59 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/1445596395' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:00.222 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:00 smithi012 ceph-mon[91852]: pgmap v164: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:00.223 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:00 smithi012 ceph-mon[95978]: pgmap v164: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:00.473 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:00 smithi012 systemd[1]: Stopping Ceph osd.2 for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T05:04:00.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:00 smithi116 ceph-mon[85502]: pgmap v164: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:00.763 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:00 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2[52883]: 2024-04-26T05:04:00.472+0000 7f0e02260700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:04:00.763 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:00 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2[52883]: 2024-04-26T05:04:00.472+0000 7f0e02260700 -1 osd.2 90 *** Got signal Terminated *** 2024-04-26T05:04:00.763 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:00 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2[52883]: 2024-04-26T05:04:00.472+0000 7f0e02260700 -1 osd.2 90 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:04:01.392 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:01 smithi012 ceph-mon[91852]: osd.2 marked itself down and dead 2024-04-26T05:04:01.392 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:01 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/548371120' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:01.392 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:01 smithi012 ceph-mon[95978]: osd.2 marked itself down and dead 2024-04-26T05:04:01.392 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:01 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/548371120' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:01.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:01 smithi116 ceph-mon[85502]: osd.2 marked itself down and dead 2024-04-26T05:04:01.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:01 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/548371120' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:01.700 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:01 smithi012 podman[113142]: 2024-04-26 05:04:01.391495111 +0000 UTC m=+1.022027985 container died 432af63060b81eb8ef26258588846b9c1e628adb0b8d0ce421d9e00efa1ef69c (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=quincy-c61472c, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-04-26T05:04:02.013 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:01 smithi012 podman[113142]: 2024-04-26 05:04:01.78956758 +0000 UTC m=+1.420100438 container cleanup 432af63060b81eb8ef26258588846b9c1e628adb0b8d0ce421d9e00efa1ef69c (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=quincy-c61472c, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240424) 2024-04-26T05:04:02.013 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:01 smithi012 bash[113142]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2 2024-04-26T05:04:02.367 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:02 smithi012 podman[113186]: 2024-04-26 05:04:02.029430542 +0000 UTC m=+0.636079154 container remove 432af63060b81eb8ef26258588846b9c1e628adb0b8d0ce421d9e00efa1ef69c (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=quincy-c61472c, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-04-26T05:04:02.367 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:02 smithi012 ceph-mon[91852]: pgmap v165: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:02.367 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:02 smithi012 ceph-mon[91852]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T05:04:02.367 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:02 smithi012 ceph-mon[91852]: osdmap e91: 8 total, 7 up, 8 in 2024-04-26T05:04:02.368 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:02 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:02.368 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:02 smithi012 ceph-mon[95978]: pgmap v165: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:02.368 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:02 smithi012 ceph-mon[95978]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T05:04:02.368 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:02 smithi012 ceph-mon[95978]: osdmap e91: 8 total, 7 up, 8 in 2024-04-26T05:04:02.368 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:02 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:02.393 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:02 smithi116 ceph-mon[85502]: pgmap v165: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:02.393 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:02 smithi116 ceph-mon[85502]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T05:04:02.393 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:02 smithi116 ceph-mon[85502]: osdmap e91: 8 total, 7 up, 8 in 2024-04-26T05:04:02.393 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:02 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:02.745 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:02 smithi012 podman[113283]: 2024-04-26 05:04:02.366840474 +0000 UTC m=+0.106985867 container create 4730e809b0647221f3619abf4958f68b69060348521c269915076a0b69e67b6a (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T05:04:02.745 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:02 smithi012 podman[113283]: 2024-04-26 05:04:02.278577287 +0000 UTC m=+0.018722693 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:04:03.013 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:02 smithi012 podman[113283]: 2024-04-26 05:04:02.744521967 +0000 UTC m=+0.484667357 container init 4730e809b0647221f3619abf4958f68b69060348521c269915076a0b69e67b6a (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2-deactivate, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb) 2024-04-26T05:04:03.013 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:02 smithi012 podman[113283]: 2024-04-26 05:04:02.747984875 +0000 UTC m=+0.488130266 container start 4730e809b0647221f3619abf4958f68b69060348521c269915076a0b69e67b6a (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0) 2024-04-26T05:04:03.013 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:02 smithi012 podman[113283]: 2024-04-26 05:04:02.836150132 +0000 UTC m=+0.576295536 container attach 4730e809b0647221f3619abf4958f68b69060348521c269915076a0b69e67b6a (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True) 2024-04-26T05:04:03.407 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:03 smithi012 ceph-mon[91852]: osdmap e92: 8 total, 7 up, 8 in 2024-04-26T05:04:03.407 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:03 smithi012 podman[113283]: 2024-04-26 05:04:03.058474452 +0000 UTC m=+0.798619834 container died 4730e809b0647221f3619abf4958f68b69060348521c269915076a0b69e67b6a (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb) 2024-04-26T05:04:03.407 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:03 smithi012 ceph-mon[95978]: osdmap e92: 8 total, 7 up, 8 in 2024-04-26T05:04:03.409 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:03 smithi116 ceph-mon[85502]: osdmap e92: 8 total, 7 up, 8 in 2024-04-26T05:04:04.013 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:03 smithi012 podman[113332]: 2024-04-26 05:04:03.690697996 +0000 UTC m=+0.626481411 container remove 4730e809b0647221f3619abf4958f68b69060348521c269915076a0b69e67b6a (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424) 2024-04-26T05:04:04.013 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:03 smithi012 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@osd.2.service: Deactivated successfully. 2024-04-26T05:04:04.013 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:03 smithi012 systemd[1]: Stopped Ceph osd.2 for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T05:04:04.013 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:03 smithi012 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@osd.2.service: Consumed 9.087s CPU time. 2024-04-26T05:04:04.013 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:03 smithi012 systemd[1]: Starting Ceph osd.2 for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T05:04:04.378 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:04 smithi012 ceph-mon[91852]: pgmap v168: 73 pgs: 1 active+undersized+degraded, 1 stale+active+clean, 12 active+undersized, 59 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-26T05:04:04.378 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:04 smithi012 ceph-mon[91852]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2024-04-26T05:04:04.378 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:04 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/2879607392' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:04.379 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:04 smithi012 ceph-mon[95978]: pgmap v168: 73 pgs: 1 active+undersized+degraded, 1 stale+active+clean, 12 active+undersized, 59 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-26T05:04:04.379 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:04 smithi012 ceph-mon[95978]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2024-04-26T05:04:04.379 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:04 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/2879607392' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:04.379 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:04 smithi012 podman[113437]: 2024-04-26 05:04:04.108579551 +0000 UTC m=+0.086815306 container create f4387453e3416b18e1bd2ecf14eab5ec0498c533c4134076a4f086aa93a3e623 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2-activate, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=) 2024-04-26T05:04:04.379 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:04 smithi012 podman[113437]: 2024-04-26 05:04:04.040529209 +0000 UTC m=+0.018764931 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:04:04.379 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:04 smithi012 podman[113437]: 2024-04-26 05:04:04.302566604 +0000 UTC m=+0.280802313 container init f4387453e3416b18e1bd2ecf14eab5ec0498c533c4134076a4f086aa93a3e623 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2-activate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424) 2024-04-26T05:04:04.379 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:04 smithi012 podman[113437]: 2024-04-26 05:04:04.306332031 +0000 UTC m=+0.284567744 container start f4387453e3416b18e1bd2ecf14eab5ec0498c533c4134076a4f086aa93a3e623 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2-activate, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-04-26T05:04:04.379 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:04 smithi012 podman[113437]: 2024-04-26 05:04:04.377618498 +0000 UTC m=+0.355854221 container attach f4387453e3416b18e1bd2ecf14eab5ec0498c533c4134076a4f086aa93a3e623 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2) 2024-04-26T05:04:04.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:04 smithi116 ceph-mon[85502]: pgmap v168: 73 pgs: 1 active+undersized+degraded, 1 stale+active+clean, 12 active+undersized, 59 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-26T05:04:04.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:04 smithi116 ceph-mon[85502]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2024-04-26T05:04:04.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:04 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/2879607392' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:05.108 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:04 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2-activate[113451]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-26T05:04:05.108 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:04 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2-activate[113451]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-26T05:04:05.108 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:04 smithi012 bash[113437]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-26T05:04:05.108 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:04 smithi012 bash[113437]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-26T05:04:05.108 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:04 smithi012 bash[113437]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-26T05:04:05.109 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:04 smithi012 bash[113437]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-26T05:04:05.109 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:04 smithi012 bash[113437]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-26T05:04:05.109 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:04 smithi012 bash[113437]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-26T05:04:05.109 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:04 smithi012 bash[113437]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-26T05:04:05.109 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:04 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2-activate[113451]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-26T05:04:05.109 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:04 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2-activate[113451]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-26T05:04:05.109 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:04 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2-activate[113451]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-26T05:04:05.109 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:04 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2-activate[113451]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-26T05:04:05.109 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:04 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2-activate[113451]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-26T05:04:05.109 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:04 smithi012 podman[113437]: 2024-04-26 05:04:04.788227324 +0000 UTC m=+0.766463034 container died f4387453e3416b18e1bd2ecf14eab5ec0498c533c4134076a4f086aa93a3e623 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2-activate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T05:04:05.735 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:05 smithi012 podman[113580]: 2024-04-26 05:04:05.365333638 +0000 UTC m=+0.566864888 container remove f4387453e3416b18e1bd2ecf14eab5ec0498c533c4134076a4f086aa93a3e623 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2-activate, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-04-26T05:04:05.736 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:05 smithi012 podman[113665]: 2024-04-26 05:04:05.485108154 +0000 UTC m=+0.018805843 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:04:05.736 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:05 smithi012 podman[113665]: 2024-04-26 05:04:05.607998875 +0000 UTC m=+0.141696561 container create 388804ce91742495d116e8ff604998919e22fe2247239619ae64e2e258fe597d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:04:05.997 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:05 smithi012 podman[113665]: 2024-04-26 05:04:05.869442493 +0000 UTC m=+0.403140198 container init 388804ce91742495d116e8ff604998919e22fe2247239619ae64e2e258fe597d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-04-26T05:04:05.998 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:05 smithi012 podman[113665]: 2024-04-26 05:04:05.872898232 +0000 UTC m=+0.406595926 container start 388804ce91742495d116e8ff604998919e22fe2247239619ae64e2e258fe597d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-04-26T05:04:05.998 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:05 smithi012 ceph-osd[113688]: -- 172.21.15.12:0/3248161796 <== mon.2 v2:172.21.15.12:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a184725680 con 0x55a1846ef000 2024-04-26T05:04:05.998 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:05 smithi012 bash[113665]: 388804ce91742495d116e8ff604998919e22fe2247239619ae64e2e258fe597d 2024-04-26T05:04:05.998 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:05 smithi012 systemd[1]: Started Ceph osd.2 for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T05:04:06.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:06 smithi012 ceph-mon[91852]: pgmap v169: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T05:04:06.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:06 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/536658479' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:06.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:06 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:06.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:06 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:06.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:06 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:04:06.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:06 smithi012 ceph-mon[95978]: pgmap v169: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T05:04:06.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:06 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/536658479' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:06.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:06 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:06.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:06 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:06.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:06 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:04:06.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:06 smithi116 ceph-mon[85502]: pgmap v169: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T05:04:06.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:06 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/536658479' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:06.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:06 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:06.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:06 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:06.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:06 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:04:06.763 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:06 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2[113683]: 2024-04-26T05:04:06.413+0000 7f6e780747c0 -1 Falling back to public interface 2024-04-26T05:04:08.133 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:08 smithi012 ceph-mon[91852]: pgmap v170: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T05:04:08.133 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:08 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/1947304683' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:08.133 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:04:07 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:04:07] "GET /metrics HTTP/1.1" 200 33960 "" "Prometheus/2.43.0" 2024-04-26T05:04:08.133 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:08 smithi012 ceph-mon[95978]: pgmap v170: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T05:04:08.133 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:08 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/1947304683' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:08.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:08 smithi116 ceph-mon[85502]: pgmap v170: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T05:04:08.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:08 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/1947304683' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:09.513 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:09 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2[113683]: 2024-04-26T05:04:09.162+0000 7f6e780747c0 -1 osd.2 90 log_to_monitors true 2024-04-26T05:04:10.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:10 smithi116 ceph-mon[85502]: pgmap v171: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T05:04:10.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:10 smithi116 ceph-mon[85502]: from='osd.2 [v2:172.21.15.12:6818/2306607800,v1:172.21.15.12:6819/2306607800]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T05:04:10.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:10 smithi116 ceph-mon[85502]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T05:04:10.512 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:04:10 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2[113683]: 2024-04-26T05:04:10.103+0000 7f6e694cc700 -1 osd.2 90 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T05:04:10.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:10 smithi012 ceph-mon[91852]: pgmap v171: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T05:04:10.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:10 smithi012 ceph-mon[91852]: from='osd.2 [v2:172.21.15.12:6818/2306607800,v1:172.21.15.12:6819/2306607800]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T05:04:10.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:10 smithi012 ceph-mon[91852]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T05:04:10.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:10 smithi012 ceph-mon[95978]: pgmap v171: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T05:04:10.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:10 smithi012 ceph-mon[95978]: from='osd.2 [v2:172.21.15.12:6818/2306607800,v1:172.21.15.12:6819/2306607800]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T05:04:10.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:10 smithi012 ceph-mon[95978]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T05:04:11.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:11 smithi116 ceph-mon[85502]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T05:04:11.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:11 smithi116 ceph-mon[85502]: osdmap e93: 8 total, 7 up, 8 in 2024-04-26T05:04:11.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:11 smithi116 ceph-mon[85502]: from='osd.2 [v2:172.21.15.12:6818/2306607800,v1:172.21.15.12:6819/2306607800]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-04-26T05:04:11.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:11 smithi116 ceph-mon[85502]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-04-26T05:04:11.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:11 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:11.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:11 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:11.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:11 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/191758784' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:11.512 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:11 smithi012 ceph-mon[95978]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T05:04:11.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:11 smithi012 ceph-mon[95978]: osdmap e93: 8 total, 7 up, 8 in 2024-04-26T05:04:11.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:11 smithi012 ceph-mon[95978]: from='osd.2 [v2:172.21.15.12:6818/2306607800,v1:172.21.15.12:6819/2306607800]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-04-26T05:04:11.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:11 smithi012 ceph-mon[95978]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-04-26T05:04:11.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:11 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:11.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:11 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:11.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:11 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/191758784' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:11.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:11 smithi012 ceph-mon[91852]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T05:04:11.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:11 smithi012 ceph-mon[91852]: osdmap e93: 8 total, 7 up, 8 in 2024-04-26T05:04:11.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:11 smithi012 ceph-mon[91852]: from='osd.2 [v2:172.21.15.12:6818/2306607800,v1:172.21.15.12:6819/2306607800]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-04-26T05:04:11.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:11 smithi012 ceph-mon[91852]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-04-26T05:04:11.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:11 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:11.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:11 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:11.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:11 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/191758784' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:12.165 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:12 smithi012 ceph-mon[91852]: pgmap v173: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T05:04:12.166 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:12 smithi012 ceph-mon[91852]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T05:04:12.166 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:12 smithi012 ceph-mon[91852]: osd.2 [v2:172.21.15.12:6818/2306607800,v1:172.21.15.12:6819/2306607800] boot 2024-04-26T05:04:12.166 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:12 smithi012 ceph-mon[91852]: osdmap e94: 8 total, 8 up, 8 in 2024-04-26T05:04:12.166 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:12 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T05:04:12.166 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:12 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:12.166 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:12 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:12.166 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:12 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:12.166 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:12 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:12.166 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:12 smithi012 ceph-mon[95978]: pgmap v173: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T05:04:12.167 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:12 smithi012 ceph-mon[95978]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T05:04:12.167 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:12 smithi012 ceph-mon[95978]: osd.2 [v2:172.21.15.12:6818/2306607800,v1:172.21.15.12:6819/2306607800] boot 2024-04-26T05:04:12.167 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:12 smithi012 ceph-mon[95978]: osdmap e94: 8 total, 8 up, 8 in 2024-04-26T05:04:12.167 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:12 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T05:04:12.167 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:12 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:12.167 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:12 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:12.167 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:12 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:12.167 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:12 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:12.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:12 smithi116 ceph-mon[85502]: pgmap v173: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T05:04:12.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:12 smithi116 ceph-mon[85502]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T05:04:12.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:12 smithi116 ceph-mon[85502]: osd.2 [v2:172.21.15.12:6818/2306607800,v1:172.21.15.12:6819/2306607800] boot 2024-04-26T05:04:12.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:12 smithi116 ceph-mon[85502]: osdmap e94: 8 total, 8 up, 8 in 2024-04-26T05:04:12.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:12 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T05:04:12.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:12 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:12.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:12 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:12.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:12 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:12.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:12 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:13.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:13 smithi012 ceph-mon[91852]: osdmap e95: 8 total, 8 up, 8 in 2024-04-26T05:04:13.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:13 smithi012 ceph-mon[91852]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T05:04:13.180 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:13 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/1543202865' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:13.180 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:13 smithi012 ceph-mon[95978]: osdmap e95: 8 total, 8 up, 8 in 2024-04-26T05:04:13.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:13 smithi012 ceph-mon[95978]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T05:04:13.181 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:13 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/1543202865' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:13.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:13 smithi116 ceph-mon[85502]: osdmap e95: 8 total, 8 up, 8 in 2024-04-26T05:04:13.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:13 smithi116 ceph-mon[85502]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T05:04:13.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:13 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/1543202865' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:14.427 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:14 smithi012 ceph-mon[91852]: pgmap v176: 73 pgs: 8 activating, 4 peering, 1 active+undersized+degraded, 6 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-26T05:04:14.428 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:14 smithi012 ceph-mon[95978]: pgmap v176: 73 pgs: 8 activating, 4 peering, 1 active+undersized+degraded, 6 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-26T05:04:14.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:14 smithi116 ceph-mon[85502]: pgmap v176: 73 pgs: 8 activating, 4 peering, 1 active+undersized+degraded, 6 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-26T05:04:15.120 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:15 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/1727003897' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:15.120 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:15 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:15.121 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:15 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:15.121 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:15 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:04:15.121 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:15 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:04:15.121 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:15 smithi012 ceph-mon[91852]: pgmap v177: 73 pgs: 8 activating, 4 peering, 2 active+undersized, 59 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:15.121 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:15 smithi012 ceph-mon[91852]: pgmap v178: 73 pgs: 8 activating, 4 peering, 2 active+undersized, 59 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:15.121 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:15 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:15.121 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:15 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:04:15.121 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:15 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:04:15.121 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:15 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:04:15.121 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:15 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:04:15.121 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:15 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T05:04:15.122 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:04:15 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T05:04:15.072+0000 7f0e1e4f1700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-04-26T05:04:15.122 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:15 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/1727003897' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:15.122 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:15 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:15.122 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:15 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:15.122 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:15 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:04:15.122 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:15 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:04:15.122 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:15 smithi012 ceph-mon[95978]: pgmap v177: 73 pgs: 8 activating, 4 peering, 2 active+undersized, 59 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:15.122 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:15 smithi012 ceph-mon[95978]: pgmap v178: 73 pgs: 8 activating, 4 peering, 2 active+undersized, 59 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:15.122 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:15 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:15.122 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:15 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:04:15.122 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:15 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:04:15.123 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:15 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:04:15.123 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:15 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:04:15.123 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:15 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T05:04:15.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:15 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/1727003897' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:15.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:15 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:15.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:15 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:15.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:15 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:04:15.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:15 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:04:15.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:15 smithi116 ceph-mon[85502]: pgmap v177: 73 pgs: 8 activating, 4 peering, 2 active+undersized, 59 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:15.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:15 smithi116 ceph-mon[85502]: pgmap v178: 73 pgs: 8 activating, 4 peering, 2 active+undersized, 59 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:15.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:15 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:15.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:15 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:04:15.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:15 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:04:15.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:15 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:04:15.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:15 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:04:15.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:15 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T05:04:16.459 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:16 smithi012 ceph-mon[91852]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T05:04:16.459 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:16 smithi012 ceph-mon[91852]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-04-26T05:04:16.459 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:16 smithi012 ceph-mon[91852]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2024-04-26T05:04:16.459 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:16 smithi012 ceph-mon[95978]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T05:04:16.460 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:16 smithi012 ceph-mon[95978]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-04-26T05:04:16.460 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:16 smithi012 ceph-mon[95978]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2024-04-26T05:04:16.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:16 smithi116 ceph-mon[85502]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T05:04:16.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:16 smithi116 ceph-mon[85502]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-04-26T05:04:16.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:16 smithi116 ceph-mon[85502]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2024-04-26T05:04:17.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:17 smithi116 ceph-mon[85502]: pgmap v179: 73 pgs: 8 activating, 4 peering, 61 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:17.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:17 smithi012 ceph-mon[91852]: pgmap v179: 73 pgs: 8 activating, 4 peering, 61 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:17.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:17 smithi012 ceph-mon[95978]: pgmap v179: 73 pgs: 8 activating, 4 peering, 61 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:18.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:18 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/549248598' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:18.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:04:17 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:04:17] "GET /metrics HTTP/1.1" 200 33805 "" "Prometheus/2.43.0" 2024-04-26T05:04:18.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:18 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/549248598' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:18.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:18 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/549248598' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:19.494 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:19 smithi116 ceph-mon[85502]: pgmap v180: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:19.512 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:19 smithi012 ceph-mon[91852]: pgmap v180: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:19.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:19 smithi012 ceph-mon[95978]: pgmap v180: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:20.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:20 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/3517234909' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:20.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:20 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/3517234909' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:20.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:20 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/3517234909' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:21.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:21 smithi116 ceph-mon[85502]: pgmap v181: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:21.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:21 smithi012 ceph-mon[91852]: pgmap v181: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:21.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:21 smithi012 ceph-mon[95978]: pgmap v181: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:22.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:22 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/4277756980' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:22.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:22 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/4277756980' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:22.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:22 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/4277756980' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:23.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:23 smithi116 ceph-mon[85502]: pgmap v182: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:23.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:23 smithi012 ceph-mon[91852]: pgmap v182: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:23.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:23 smithi012 ceph-mon[95978]: pgmap v182: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:24.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:24 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/1167801572' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:24.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:24 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/1167801572' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:24.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:24 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/1167801572' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:25.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:25 smithi116 ceph-mon[85502]: pgmap v183: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:25.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:25 smithi012 ceph-mon[91852]: pgmap v183: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:25.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:25 smithi012 ceph-mon[95978]: pgmap v183: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:27.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:27 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:27.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:27 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:27.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:27 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:27.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:27 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/1798650916' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:27.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:27 smithi116 ceph-mon[85502]: pgmap v184: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:27.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:27 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:27.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:27 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:27.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:27 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:27.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:27 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/1798650916' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:27.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:27 smithi012 ceph-mon[91852]: pgmap v184: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:27.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:27 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:27.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:27 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:27.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:27 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:27.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:27 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/1798650916' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:27.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:27 smithi012 ceph-mon[95978]: pgmap v184: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:28.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:04:27 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:04:27] "GET /metrics HTTP/1.1" 200 33954 "" "Prometheus/2.43.0" 2024-04-26T05:04:29.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:28 smithi012 ceph-mon[95978]: pgmap v185: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:29.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:28 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/4036058135' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:29.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:28 smithi012 ceph-mon[91852]: pgmap v185: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:29.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:28 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/4036058135' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:29.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:28 smithi116 ceph-mon[85502]: pgmap v185: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:29.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:28 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/4036058135' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:30.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:30 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T05:04:30.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:30 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T05:04:30.493 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:30 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T05:04:31.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:31 smithi116 ceph-mon[85502]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T05:04:31.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:31 smithi116 ceph-mon[85502]: Upgrade: osd.3 is safe to restart 2024-04-26T05:04:31.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:31 smithi116 ceph-mon[85502]: pgmap v186: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:31.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:31 smithi012 ceph-mon[91852]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T05:04:31.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:31 smithi012 ceph-mon[91852]: Upgrade: osd.3 is safe to restart 2024-04-26T05:04:31.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:31 smithi012 ceph-mon[91852]: pgmap v186: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:31.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:31 smithi012 ceph-mon[95978]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T05:04:31.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:31 smithi012 ceph-mon[95978]: Upgrade: osd.3 is safe to restart 2024-04-26T05:04:31.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:31 smithi012 ceph-mon[95978]: pgmap v186: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:32.397 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:32 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/3600502264' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:32.397 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:32 smithi012 ceph-mon[91852]: Upgrade: Updating osd.3 2024-04-26T05:04:32.398 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:32 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:32.398 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:32 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T05:04:32.398 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:32 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:04:32.398 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:32 smithi012 ceph-mon[91852]: Deploying daemon osd.3 on smithi012 2024-04-26T05:04:32.398 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:32 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/3600502264' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:32.398 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:32 smithi012 ceph-mon[95978]: Upgrade: Updating osd.3 2024-04-26T05:04:32.398 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:32 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:32.398 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:32 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T05:04:32.398 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:32 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:04:32.398 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:32 smithi012 ceph-mon[95978]: Deploying daemon osd.3 on smithi012 2024-04-26T05:04:32.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:32 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/3600502264' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:32.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:32 smithi116 ceph-mon[85502]: Upgrade: Updating osd.3 2024-04-26T05:04:32.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:32 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:32.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:32 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T05:04:32.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:32 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:04:32.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:32 smithi116 ceph-mon[85502]: Deploying daemon osd.3 on smithi012 2024-04-26T05:04:33.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:33 smithi012 ceph-mon[91852]: pgmap v187: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:33.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:33 smithi012 ceph-mon[95978]: pgmap v187: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:33.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:33 smithi116 ceph-mon[85502]: pgmap v187: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:34.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:34 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/3933245282' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:34.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:34 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/3933245282' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:34.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:34 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/3933245282' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:35.189 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:35 smithi012 ceph-mon[91852]: pgmap v188: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:35.189 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:35 smithi012 ceph-mon[95978]: pgmap v188: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:35.189 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:34 smithi012 systemd[1]: Stopping Ceph osd.3 for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T05:04:35.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:35 smithi116 ceph-mon[85502]: pgmap v188: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:35.512 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:35 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3[58788]: 2024-04-26T05:04:35.187+0000 7f758684f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:04:35.513 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:35 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3[58788]: 2024-04-26T05:04:35.187+0000 7f758684f700 -1 osd.3 95 *** Got signal Terminated *** 2024-04-26T05:04:35.513 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:35 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3[58788]: 2024-04-26T05:04:35.187+0000 7f758684f700 -1 osd.3 95 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:04:36.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:36 smithi116 ceph-mon[85502]: osd.3 marked itself down and dead 2024-04-26T05:04:36.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:36 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/1643931446' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:36.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:36 smithi012 ceph-mon[95978]: osd.3 marked itself down and dead 2024-04-26T05:04:36.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:36 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/1643931446' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:36.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:36 smithi012 ceph-mon[91852]: osd.3 marked itself down and dead 2024-04-26T05:04:36.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:36 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/1643931446' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:36.861 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:36 smithi012 podman[118044]: 2024-04-26 05:04:36.555780731 +0000 UTC m=+1.470798805 container died f65d4289a97007a14f05b47056d3842b798e7f58ce9fd0a2586f9ebc97553c55 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=quincy-c61472c, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-04-26T05:04:37.159 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:36 smithi012 podman[118044]: 2024-04-26 05:04:36.962291682 +0000 UTC m=+1.877309755 container cleanup f65d4289a97007a14f05b47056d3842b798e7f58ce9fd0a2586f9ebc97553c55 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=quincy-c61472c, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2) 2024-04-26T05:04:37.160 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:36 smithi012 bash[118044]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3 2024-04-26T05:04:37.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:37 smithi116 ceph-mon[85502]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T05:04:37.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:37 smithi116 ceph-mon[85502]: osdmap e96: 8 total, 7 up, 8 in 2024-04-26T05:04:37.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:37 smithi116 ceph-mon[85502]: pgmap v190: 73 pgs: 13 stale+active+clean, 60 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:37.512 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:37 smithi012 ceph-mon[91852]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T05:04:37.512 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:37 smithi012 ceph-mon[91852]: osdmap e96: 8 total, 7 up, 8 in 2024-04-26T05:04:37.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:37 smithi012 ceph-mon[91852]: pgmap v190: 73 pgs: 13 stale+active+clean, 60 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:37.513 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:37 smithi012 podman[118087]: 2024-04-26 05:04:37.214070506 +0000 UTC m=+0.652351211 container remove f65d4289a97007a14f05b47056d3842b798e7f58ce9fd0a2586f9ebc97553c55 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, RELEASE=quincy-c61472c) 2024-04-26T05:04:37.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:37 smithi012 ceph-mon[95978]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T05:04:37.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:37 smithi012 ceph-mon[95978]: osdmap e96: 8 total, 7 up, 8 in 2024-04-26T05:04:37.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:37 smithi012 ceph-mon[95978]: pgmap v190: 73 pgs: 13 stale+active+clean, 60 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:37.818 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:37 smithi012 podman[118175]: 2024-04-26 05:04:37.434031976 +0000 UTC m=+0.018941579 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:04:37.818 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:37 smithi012 podman[118175]: 2024-04-26 05:04:37.548571896 +0000 UTC m=+0.133481512 container create 2e028cc0fd8d2d88397284b360c528a84591c6f71acb81b37d37a3aff65b4b0d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424) 2024-04-26T05:04:37.818 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:37 smithi012 podman[118175]: 2024-04-26 05:04:37.75931044 +0000 UTC m=+0.344220016 container init 2e028cc0fd8d2d88397284b360c528a84591c6f71acb81b37d37a3aff65b4b0d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True) 2024-04-26T05:04:37.818 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:37 smithi012 podman[118175]: 2024-04-26 05:04:37.762509325 +0000 UTC m=+0.347418904 container start 2e028cc0fd8d2d88397284b360c528a84591c6f71acb81b37d37a3aff65b4b0d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-04-26T05:04:37.818 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:37 smithi012 podman[118175]: 2024-04-26 05:04:37.817331676 +0000 UTC m=+0.402241257 container attach 2e028cc0fd8d2d88397284b360c528a84591c6f71acb81b37d37a3aff65b4b0d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3-deactivate, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424) 2024-04-26T05:04:38.092 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:04:37 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:04:37] "GET /metrics HTTP/1.1" 200 33954 "" "Prometheus/2.43.0" 2024-04-26T05:04:38.374 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:38 smithi012 ceph-mon[91852]: osdmap e97: 8 total, 7 up, 8 in 2024-04-26T05:04:38.374 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:38 smithi012 ceph-mon[95978]: osdmap e97: 8 total, 7 up, 8 in 2024-04-26T05:04:38.375 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:38 smithi012 podman[118175]: 2024-04-26 05:04:38.092214029 +0000 UTC m=+0.677123615 container died 2e028cc0fd8d2d88397284b360c528a84591c6f71acb81b37d37a3aff65b4b0d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2) 2024-04-26T05:04:38.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:38 smithi116 ceph-mon[85502]: osdmap e97: 8 total, 7 up, 8 in 2024-04-26T05:04:39.013 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:38 smithi012 podman[118224]: 2024-04-26 05:04:38.730310187 +0000 UTC m=+0.629977008 container remove 2e028cc0fd8d2d88397284b360c528a84591c6f71acb81b37d37a3aff65b4b0d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3-deactivate, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb) 2024-04-26T05:04:39.013 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:38 smithi012 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@osd.3.service: Deactivated successfully. 2024-04-26T05:04:39.013 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:38 smithi012 systemd[1]: Stopped Ceph osd.3 for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T05:04:39.014 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:38 smithi012 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@osd.3.service: Consumed 5.313s CPU time. 2024-04-26T05:04:39.014 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:38 smithi012 systemd[1]: Starting Ceph osd.3 for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T05:04:39.434 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:39 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/1106859854' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:39.434 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:39 smithi012 ceph-mon[91852]: pgmap v192: 73 pgs: 1 active+undersized+degraded, 14 peering, 5 stale+active+clean, 6 active+undersized, 47 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2024-04-26T05:04:39.435 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:39 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/1106859854' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:39.435 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:39 smithi012 ceph-mon[95978]: pgmap v192: 73 pgs: 1 active+undersized+degraded, 14 peering, 5 stale+active+clean, 6 active+undersized, 47 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2024-04-26T05:04:39.435 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:39 smithi012 podman[118340]: 2024-04-26 05:04:39.096818092 +0000 UTC m=+0.019647071 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:04:39.435 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:39 smithi012 podman[118340]: 2024-04-26 05:04:39.198040235 +0000 UTC m=+0.120869184 container create b439de543e704c0c7583407b4008987aa4ab215d734d87139e7286c86480327e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3-activate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-04-26T05:04:39.435 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:39 smithi012 podman[118340]: 2024-04-26 05:04:39.433993522 +0000 UTC m=+0.356822480 container init b439de543e704c0c7583407b4008987aa4ab215d734d87139e7286c86480327e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:04:39.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:39 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/1106859854' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:39.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:39 smithi116 ceph-mon[85502]: pgmap v192: 73 pgs: 1 active+undersized+degraded, 14 peering, 5 stale+active+clean, 6 active+undersized, 47 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2024-04-26T05:04:39.763 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:39 smithi012 podman[118340]: 2024-04-26 05:04:39.437598339 +0000 UTC m=+0.360427294 container start b439de543e704c0c7583407b4008987aa4ab215d734d87139e7286c86480327e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3-activate, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-04-26T05:04:39.763 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:39 smithi012 podman[118340]: 2024-04-26 05:04:39.525635039 +0000 UTC m=+0.448464002 container attach b439de543e704c0c7583407b4008987aa4ab215d734d87139e7286c86480327e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3-activate, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True) 2024-04-26T05:04:40.176 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:39 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3-activate[118356]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-26T05:04:40.176 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:39 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3-activate[118356]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-26T05:04:40.176 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:39 smithi012 bash[118340]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-26T05:04:40.176 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:39 smithi012 bash[118340]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-26T05:04:40.176 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:39 smithi012 bash[118340]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-26T05:04:40.176 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:39 smithi012 bash[118340]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-26T05:04:40.176 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:39 smithi012 bash[118340]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-26T05:04:40.176 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:39 smithi012 bash[118340]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-26T05:04:40.176 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:39 smithi012 bash[118340]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-26T05:04:40.176 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:39 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3-activate[118356]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-26T05:04:40.176 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:39 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3-activate[118356]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-26T05:04:40.177 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:39 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3-activate[118356]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-26T05:04:40.177 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:39 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3-activate[118356]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-26T05:04:40.177 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:39 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3-activate[118356]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-26T05:04:40.177 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:39 smithi012 podman[118340]: 2024-04-26 05:04:39.921956935 +0000 UTC m=+0.844785897 container died b439de543e704c0c7583407b4008987aa4ab215d734d87139e7286c86480327e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3-activate, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-26T05:04:40.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:40 smithi116 ceph-mon[85502]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-04-26T05:04:40.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:40 smithi116 ceph-mon[85502]: Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2024-04-26T05:04:40.496 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:40 smithi012 ceph-mon[91852]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-04-26T05:04:40.497 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:40 smithi012 ceph-mon[91852]: Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2024-04-26T05:04:40.497 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:40 smithi012 ceph-mon[95978]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-04-26T05:04:40.497 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:40 smithi012 ceph-mon[95978]: Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2024-04-26T05:04:40.763 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:40 smithi012 podman[118475]: 2024-04-26 05:04:40.496521389 +0000 UTC m=+0.569398430 container remove b439de543e704c0c7583407b4008987aa4ab215d734d87139e7286c86480327e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3-activate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424) 2024-04-26T05:04:40.763 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:40 smithi012 podman[118563]: 2024-04-26 05:04:40.623241275 +0000 UTC m=+0.025756324 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:04:40.763 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:40 smithi012 podman[118563]: 2024-04-26 05:04:40.73095233 +0000 UTC m=+0.133467365 container create a25aba82b703903a56ff4555b8df31635fa7f0f5b1e8efd592125fcb98e26e61 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-04-26T05:04:41.185 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:40 smithi012 podman[118563]: 2024-04-26 05:04:40.959278844 +0000 UTC m=+0.361793894 container init a25aba82b703903a56ff4555b8df31635fa7f0f5b1e8efd592125fcb98e26e61 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True) 2024-04-26T05:04:41.186 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:40 smithi012 podman[118563]: 2024-04-26 05:04:40.962707645 +0000 UTC m=+0.365222685 container start a25aba82b703903a56ff4555b8df31635fa7f0f5b1e8efd592125fcb98e26e61 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-04-26T05:04:41.186 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:40 smithi012 ceph-osd[118586]: -- 172.21.15.12:0/2466104422 <== mon.2 v2:172.21.15.12:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5613197df680 con 0x5613197a8c00 2024-04-26T05:04:41.186 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:41 smithi012 bash[118563]: a25aba82b703903a56ff4555b8df31635fa7f0f5b1e8efd592125fcb98e26e61 2024-04-26T05:04:41.186 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:41 smithi012 systemd[1]: Started Ceph osd.3 for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T05:04:41.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:41 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/3676110566' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:41.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:41 smithi116 ceph-mon[85502]: pgmap v193: 73 pgs: 1 active+undersized+degraded, 14 peering, 16 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2024-04-26T05:04:41.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:41 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:41.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:41 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:41.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:41 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:04:41.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:41 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:41.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:41 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:41.504 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:41 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/3676110566' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:41.504 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:41 smithi012 ceph-mon[91852]: pgmap v193: 73 pgs: 1 active+undersized+degraded, 14 peering, 16 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2024-04-26T05:04:41.504 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:41 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:41.504 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:41 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:41.504 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:41 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:04:41.505 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:41 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:41.505 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:41 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:41.505 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:41 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/3676110566' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:41.505 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:41 smithi012 ceph-mon[95978]: pgmap v193: 73 pgs: 1 active+undersized+degraded, 14 peering, 16 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2024-04-26T05:04:41.505 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:41 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:41.505 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:41 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:41.505 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:41 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:04:41.505 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:41 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:41.505 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:41 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:41.762 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:41 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3[118582]: 2024-04-26T05:04:41.502+0000 7f791f5b07c0 -1 Falling back to public interface 2024-04-26T05:04:43.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:43 smithi012 ceph-mon[91852]: pgmap v194: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T05:04:43.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:43 smithi012 ceph-mon[91852]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2024-04-26T05:04:43.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:43 smithi012 ceph-mon[95978]: pgmap v194: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T05:04:43.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:43 smithi012 ceph-mon[95978]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2024-04-26T05:04:43.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:43 smithi116 ceph-mon[85502]: pgmap v194: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T05:04:43.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:43 smithi116 ceph-mon[85502]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2024-04-26T05:04:44.233 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:44 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/791189255' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:44.233 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:44 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/791189255' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:44.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:44 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/791189255' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:44.513 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:44 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3[118582]: 2024-04-26T05:04:44.231+0000 7f791f5b07c0 -1 osd.3 95 log_to_monitors true 2024-04-26T05:04:45.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:45 smithi012 ceph-mon[91852]: from='osd.3 [v2:172.21.15.12:6826/38281816,v1:172.21.15.12:6827/38281816]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T05:04:45.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:45 smithi012 ceph-mon[91852]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T05:04:45.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:45 smithi012 ceph-mon[91852]: pgmap v195: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T05:04:45.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:45 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:45.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:45 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:45.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:45 smithi012 ceph-mon[95978]: from='osd.3 [v2:172.21.15.12:6826/38281816,v1:172.21.15.12:6827/38281816]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T05:04:45.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:45 smithi012 ceph-mon[95978]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T05:04:45.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:45 smithi012 ceph-mon[95978]: pgmap v195: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T05:04:45.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:45 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:45.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:45 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:45.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:45 smithi116 ceph-mon[85502]: from='osd.3 [v2:172.21.15.12:6826/38281816,v1:172.21.15.12:6827/38281816]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T05:04:45.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:45 smithi116 ceph-mon[85502]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T05:04:45.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:45 smithi116 ceph-mon[85502]: pgmap v195: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T05:04:45.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:45 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:45.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:45 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:45.763 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:04:45 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3[118582]: 2024-04-26T05:04:45.499+0000 7f790ca00700 -1 osd.3 95 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T05:04:46.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:46 smithi116 ceph-mon[85502]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T05:04:46.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:46 smithi116 ceph-mon[85502]: osdmap e98: 8 total, 7 up, 8 in 2024-04-26T05:04:46.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:46 smithi116 ceph-mon[85502]: from='osd.3 [v2:172.21.15.12:6826/38281816,v1:172.21.15.12:6827/38281816]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-04-26T05:04:46.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:46 smithi116 ceph-mon[85502]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-04-26T05:04:46.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:46 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/3584046684' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:46.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:46 smithi116 ceph-mon[85502]: from='osd.3 ' entity='osd.3' 2024-04-26T05:04:46.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:46 smithi012 ceph-mon[95978]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T05:04:46.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:46 smithi012 ceph-mon[95978]: osdmap e98: 8 total, 7 up, 8 in 2024-04-26T05:04:46.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:46 smithi012 ceph-mon[95978]: from='osd.3 [v2:172.21.15.12:6826/38281816,v1:172.21.15.12:6827/38281816]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-04-26T05:04:46.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:46 smithi012 ceph-mon[95978]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-04-26T05:04:46.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:46 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/3584046684' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:46.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:46 smithi012 ceph-mon[95978]: from='osd.3 ' entity='osd.3' 2024-04-26T05:04:46.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:46 smithi012 ceph-mon[91852]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T05:04:46.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:46 smithi012 ceph-mon[91852]: osdmap e98: 8 total, 7 up, 8 in 2024-04-26T05:04:46.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:46 smithi012 ceph-mon[91852]: from='osd.3 [v2:172.21.15.12:6826/38281816,v1:172.21.15.12:6827/38281816]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-04-26T05:04:46.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:46 smithi012 ceph-mon[91852]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-04-26T05:04:46.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:46 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/3584046684' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:46.514 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:46 smithi012 ceph-mon[91852]: from='osd.3 ' entity='osd.3' 2024-04-26T05:04:47.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:47 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:47.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:47 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:47.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:47 smithi012 ceph-mon[95978]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T05:04:47.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:47 smithi012 ceph-mon[95978]: osd.3 [v2:172.21.15.12:6826/38281816,v1:172.21.15.12:6827/38281816] boot 2024-04-26T05:04:47.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:47 smithi012 ceph-mon[95978]: osdmap e99: 8 total, 8 up, 8 in 2024-04-26T05:04:47.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:47 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T05:04:47.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:47 smithi012 ceph-mon[95978]: pgmap v198: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T05:04:47.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:47 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:47.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:47 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:47.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:47 smithi012 ceph-mon[91852]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T05:04:47.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:47 smithi012 ceph-mon[91852]: osd.3 [v2:172.21.15.12:6826/38281816,v1:172.21.15.12:6827/38281816] boot 2024-04-26T05:04:47.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:47 smithi012 ceph-mon[91852]: osdmap e99: 8 total, 8 up, 8 in 2024-04-26T05:04:47.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:47 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T05:04:47.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:47 smithi012 ceph-mon[91852]: pgmap v198: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T05:04:47.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:47 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:47.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:47 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:47.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:47 smithi116 ceph-mon[85502]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T05:04:47.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:47 smithi116 ceph-mon[85502]: osd.3 [v2:172.21.15.12:6826/38281816,v1:172.21.15.12:6827/38281816] boot 2024-04-26T05:04:47.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:47 smithi116 ceph-mon[85502]: osdmap e99: 8 total, 8 up, 8 in 2024-04-26T05:04:47.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:47 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T05:04:47.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:47 smithi116 ceph-mon[85502]: pgmap v198: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T05:04:47.712 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 4 2024-04-26T05:04:48.013 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:04:47 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:04:47] "GET /metrics HTTP/1.1" 200 34024 "" "Prometheus/2.43.0" 2024-04-26T05:04:48.312 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:48 smithi012 ceph-mon[95978]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2024-04-26T05:04:48.312 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:48 smithi012 ceph-mon[95978]: osdmap e100: 8 total, 8 up, 8 in 2024-04-26T05:04:48.312 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:48 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/1904444791' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:48.312 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:48 smithi012 ceph-mon[91852]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2024-04-26T05:04:48.312 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:48 smithi012 ceph-mon[91852]: osdmap e100: 8 total, 8 up, 8 in 2024-04-26T05:04:48.312 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:48 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/1904444791' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:48.337 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade pause' 2024-04-26T05:04:48.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:48 smithi116 ceph-mon[85502]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2024-04-26T05:04:48.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:48 smithi116 ceph-mon[85502]: osdmap e100: 8 total, 8 up, 8 in 2024-04-26T05:04:48.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:48 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/1904444791' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T05:04:48.849 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:04:49.424 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:49 smithi012 ceph-mon[91852]: pgmap v200: 73 pgs: 8 peering, 2 active+undersized+degraded, 16 active+undersized, 47 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-26T05:04:49.424 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:49 smithi012 ceph-mon[95978]: pgmap v200: 73 pgs: 8 peering, 2 active+undersized+degraded, 16 active+undersized, 47 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-26T05:04:49.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:49 smithi116 ceph-mon[85502]: pgmap v200: 73 pgs: 8 peering, 2 active+undersized+degraded, 16 active+undersized, 47 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-26T05:04:50.832 INFO:teuthology.orchestra.run.smithi012.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:04:51.562 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-26T05:04:51.827 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:04:52.087 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:51 smithi012 ceph-mon[91852]: from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:04:52.087 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:51 smithi012 ceph-mon[91852]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:04:52.088 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:51 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:52.088 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:51 smithi012 ceph-mon[91852]: pgmap v201: 73 pgs: 8 peering, 65 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:52.088 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:51 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:52.088 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:51 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:52.088 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:51 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:04:52.088 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:51 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:04:52.088 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:51 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:52.088 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:51 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:04:52.088 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:51 smithi012 ceph-mon[95978]: from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:04:52.088 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:51 smithi012 ceph-mon[95978]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:04:52.088 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:51 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:52.088 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:51 smithi012 ceph-mon[95978]: pgmap v201: 73 pgs: 8 peering, 65 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:52.088 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:51 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:52.089 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:51 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:52.089 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:51 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:04:52.089 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:51 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:04:52.089 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:51 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:52.089 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:51 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:04:52.166 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:51 smithi116 ceph-mon[85502]: from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:04:52.166 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:51 smithi116 ceph-mon[85502]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:04:52.166 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:51 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:52.166 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:51 smithi116 ceph-mon[85502]: pgmap v201: 73 pgs: 8 peering, 65 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:52.166 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:51 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:52.166 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:51 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:52.166 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:51 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:04:52.166 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:51 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:04:52.166 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:51 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:52.166 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:51 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:04:52.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:52 smithi116 ceph-mon[85502]: pgmap v202: 73 pgs: 8 peering, 65 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:52.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:52 smithi116 ceph-mon[85502]: pgmap v203: 73 pgs: 8 peering, 65 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:52.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:52 smithi116 ceph-mon[85502]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2024-04-26T05:04:53.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:52 smithi012 ceph-mon[91852]: pgmap v202: 73 pgs: 8 peering, 65 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:53.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:52 smithi012 ceph-mon[91852]: pgmap v203: 73 pgs: 8 peering, 65 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:53.014 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:52 smithi012 ceph-mon[91852]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2024-04-26T05:04:53.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:52 smithi012 ceph-mon[95978]: pgmap v202: 73 pgs: 8 peering, 65 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:53.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:52 smithi012 ceph-mon[95978]: pgmap v203: 73 pgs: 8 peering, 65 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:53.015 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:52 smithi012 ceph-mon[95978]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2024-04-26T05:04:53.635 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T05:04:53.635 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.a smithi012 *:9093,9094 running (6m) 8s ago 10m 22.8M - 0.25.0 c8568f914cd2 217bb19e153b 2024-04-26T05:04:53.635 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.a smithi116 *:3000 error 63s ago 10m - - 2024-04-26T05:04:53.635 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foo.smithi012.fdqwmo smithi012 running (9m) 8s ago 9m 17.4M - 17.2.7-902-gc61472c8 0de407c1b3af 53f6c82f01b4 2024-04-26T05:04:53.635 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foo.smithi116.kduyef smithi116 running (9m) 63s ago 9m 16.2M - 17.2.7-902-gc61472c8 0de407c1b3af 4c0a0b911593 2024-04-26T05:04:53.635 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.x smithi116 *:8443,9283,8765 running (5m) 63s ago 14m 452M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 5499bf989999 2024-04-26T05:04:53.635 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.y smithi012 *:8443,9283,8765 running (6m) 8s ago 16m 507M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 3a46d0031772 2024-04-26T05:04:53.635 INFO:teuthology.orchestra.run.smithi012.stdout:mon.a smithi012 running (5m) 8s ago 16m 50.6M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 db1f873383b0 2024-04-26T05:04:53.636 INFO:teuthology.orchestra.run.smithi012.stdout:mon.b smithi116 running (3m) 63s ago 15m 34.1M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 a0c4cbd180dc 2024-04-26T05:04:53.636 INFO:teuthology.orchestra.run.smithi012.stdout:mon.c smithi012 running (4m) 8s ago 14m 40.0M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 4d75f0bd2f16 2024-04-26T05:04:53.636 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.a smithi012 *:9100 running (6m) 8s ago 10m 16.4M - 1.5.0 0da6a335fe13 6390338999c1 2024-04-26T05:04:53.636 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.b smithi116 *:9100 running (6m) 63s ago 10m 16.5M - 1.5.0 0da6a335fe13 7d0b1e809531 2024-04-26T05:04:53.636 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (2m) 8s ago 14m 40.2M 2481M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 512ac46479c2 2024-04-26T05:04:53.636 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (67s) 8s ago 13m 40.1M 2481M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 793612ebfa66 2024-04-26T05:04:53.636 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (48s) 8s ago 13m 38.0M 2481M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 388804ce9174 2024-04-26T05:04:53.636 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (13s) 8s ago 13m 14.9M 2481M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 a25aba82b703 2024-04-26T05:04:53.636 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi116 running (12m) 63s ago 12m 48.1M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 6b0bbf77cb39 2024-04-26T05:04:53.636 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi116 running (12m) 63s ago 12m 53.2M 2480M 17.2.7-902-gc61472c8 0de407c1b3af a551ef15fa7e 2024-04-26T05:04:53.636 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi116 running (12m) 63s ago 12m 53.5M 2480M 17.2.7-902-gc61472c8 0de407c1b3af cdc0eb6f4fa2 2024-04-26T05:04:53.636 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi116 running (11m) 63s ago 11m 52.6M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 3f6689213fda 2024-04-26T05:04:53.636 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.a smithi116 *:9095 running (5m) 63s ago 11m 59.8M - 2.43.0 a07b618ecd1d e9e5b93523ae 2024-04-26T05:04:54.295 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-26T05:04:54.561 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:04:54.979 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:54 smithi012 ceph-mon[91852]: pgmap v204: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:54.979 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:54 smithi012 ceph-mon[91852]: from='client.44348 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:04:54.979 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:54 smithi012 ceph-mon[95978]: pgmap v204: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:54.979 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:54 smithi012 ceph-mon[95978]: from='client.44348 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:04:55.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:54 smithi116 ceph-mon[85502]: pgmap v204: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:55.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:54 smithi116 ceph-mon[85502]: from='client.44348 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:04:55.919 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:55 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:55.919 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:55 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:55.919 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:55 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:04:55.920 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:55 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:04:55.920 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:55 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:55.920 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:55 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:55.920 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:55 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:55.920 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:55 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:04:55.920 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:55 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:04:55.920 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:55 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:56.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:55 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:56.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:55 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:56.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:55 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:04:56.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:55 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:04:56.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:55 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:56.392 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T05:04:56.392 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.a smithi012 *:9093,9094 running (6m) 11s ago 10m 22.8M - 0.25.0 c8568f914cd2 217bb19e153b 2024-04-26T05:04:56.392 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.a smithi116 *:3000 error 1s ago 10m - - 2024-04-26T05:04:56.392 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foo.smithi012.fdqwmo smithi012 running (9m) 11s ago 9m 17.4M - 17.2.7-902-gc61472c8 0de407c1b3af 53f6c82f01b4 2024-04-26T05:04:56.392 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foo.smithi116.kduyef smithi116 running (9m) 1s ago 9m 16.4M - 17.2.7-902-gc61472c8 0de407c1b3af 4c0a0b911593 2024-04-26T05:04:56.392 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.x smithi116 *:8443,9283,8765 running (5m) 1s ago 14m 453M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 5499bf989999 2024-04-26T05:04:56.392 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.y smithi012 *:8443,9283,8765 running (6m) 11s ago 16m 507M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 3a46d0031772 2024-04-26T05:04:56.392 INFO:teuthology.orchestra.run.smithi012.stdout:mon.a smithi012 running (5m) 11s ago 16m 50.6M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 db1f873383b0 2024-04-26T05:04:56.392 INFO:teuthology.orchestra.run.smithi012.stdout:mon.b smithi116 running (3m) 1s ago 15m 35.9M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 a0c4cbd180dc 2024-04-26T05:04:56.392 INFO:teuthology.orchestra.run.smithi012.stdout:mon.c smithi012 running (4m) 11s ago 14m 40.0M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 4d75f0bd2f16 2024-04-26T05:04:56.392 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.a smithi012 *:9100 running (6m) 11s ago 10m 16.4M - 1.5.0 0da6a335fe13 6390338999c1 2024-04-26T05:04:56.393 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.b smithi116 *:9100 running (6m) 1s ago 10m 16.3M - 1.5.0 0da6a335fe13 7d0b1e809531 2024-04-26T05:04:56.393 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (2m) 11s ago 14m 40.2M 2481M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 512ac46479c2 2024-04-26T05:04:56.393 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (70s) 11s ago 14m 40.1M 2481M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 793612ebfa66 2024-04-26T05:04:56.393 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (50s) 11s ago 13m 38.0M 2481M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 388804ce9174 2024-04-26T05:04:56.393 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (15s) 11s ago 13m 14.9M 2481M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 a25aba82b703 2024-04-26T05:04:56.393 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi116 running (12m) 1s ago 12m 50.4M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 6b0bbf77cb39 2024-04-26T05:04:56.393 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi116 running (12m) 1s ago 12m 54.1M 2480M 17.2.7-902-gc61472c8 0de407c1b3af a551ef15fa7e 2024-04-26T05:04:56.393 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi116 running (12m) 1s ago 12m 54.5M 2480M 17.2.7-902-gc61472c8 0de407c1b3af cdc0eb6f4fa2 2024-04-26T05:04:56.393 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi116 running (11m) 1s ago 11m 53.6M 2480M 17.2.7-902-gc61472c8 0de407c1b3af 3f6689213fda 2024-04-26T05:04:56.393 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.a smithi116 *:9095 running (6m) 1s ago 11m 59.8M - 2.43.0 a07b618ecd1d e9e5b93523ae 2024-04-26T05:04:57.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:56 smithi012 ceph-mon[91852]: pgmap v205: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:57.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:56 smithi012 ceph-mon[91852]: pgmap v206: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:57.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:56 smithi012 ceph-mon[91852]: pgmap v207: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:57.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:56 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:57.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:56 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:57.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:56 smithi012 ceph-mon[95978]: pgmap v205: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:57.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:56 smithi012 ceph-mon[95978]: pgmap v206: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:57.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:56 smithi012 ceph-mon[95978]: pgmap v207: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:57.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:56 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:57.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:56 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:57.111 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph versions' 2024-04-26T05:04:57.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:56 smithi116 ceph-mon[85502]: pgmap v205: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:57.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:56 smithi116 ceph-mon[85502]: pgmap v206: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:57.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:56 smithi116 ceph-mon[85502]: pgmap v207: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:57.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:56 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:04:57.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:56 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:57.379 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:04:58.164 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:57 smithi012 ceph-mon[91852]: from='client.34452 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:04:58.164 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:57 smithi012 ceph-mon[95978]: from='client.34452 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:04:58.165 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:04:57 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:04:57] "GET /metrics HTTP/1.1" 200 34020 "" "Prometheus/2.43.0" 2024-04-26T05:04:58.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:57 smithi116 ceph-mon[85502]: from='client.34452 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T05:04:59.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:58 smithi012 ceph-mon[91852]: pgmap v208: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:59.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:58 smithi012 ceph-mon[95978]: pgmap v208: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:59.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:58 smithi116 ceph-mon[85502]: pgmap v208: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:04:59.275 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-04-26T05:04:59.275 INFO:teuthology.orchestra.run.smithi012.stdout: "mon": { 2024-04-26T05:04:59.275 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 3 2024-04-26T05:04:59.275 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-04-26T05:04:59.275 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr": { 2024-04-26T05:04:59.276 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-26T05:04:59.276 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-04-26T05:04:59.276 INFO:teuthology.orchestra.run.smithi012.stdout: "osd": { 2024-04-26T05:04:59.276 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)": 4, 2024-04-26T05:04:59.276 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 4 2024-04-26T05:04:59.276 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-04-26T05:04:59.276 INFO:teuthology.orchestra.run.smithi012.stdout: "mds": { 2024-04-26T05:04:59.276 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)": 2 2024-04-26T05:04:59.276 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-04-26T05:04:59.276 INFO:teuthology.orchestra.run.smithi012.stdout: "overall": { 2024-04-26T05:04:59.276 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)": 6, 2024-04-26T05:04:59.276 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 9 2024-04-26T05:04:59.276 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-04-26T05:04:59.276 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-04-26T05:04:59.915 DEBUG:teuthology.parallel:result is None 2024-04-26T05:04:59.915 INFO:teuthology.run_tasks:Running task print... 2024-04-26T05:04:59.925 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2024-04-26T05:04:59.925 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-04-26T05:04:59.990 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-26T05:04:59.990 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-26T05:04:59.990 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph mgr dump --format=json 2024-04-26T05:05:00.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:04:59 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/3059235564' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:05:00.258 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:00.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:04:59 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/3059235564' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:05:00.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:04:59 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/3059235564' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:05:01.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:00 smithi116 ceph-mon[85502]: pgmap v209: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:01.246 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:00 smithi012 ceph-mon[91852]: pgmap v209: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:01.246 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:00 smithi012 ceph-mon[95978]: pgmap v209: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:02.104 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:05:02.403 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:02 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:05:02.403 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:02 smithi012 ceph-mon[91852]: pgmap v210: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:02.404 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:02 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/4209162342' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T05:05:02.404 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:02 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:05:02.404 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:02 smithi012 ceph-mon[95978]: pgmap v210: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:02.404 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:02 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/4209162342' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T05:05:02.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:02 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:05:02.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:02 smithi116 ceph-mon[85502]: pgmap v210: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:02.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:02 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/4209162342' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T05:05:02.731 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":39,"flags":0,"active_gid":34107,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6800","nonce":2008222090},{"type":"v1","addr":"172.21.15.12:6801","nonce":2008222090}]},"active_addr":"172.21.15.12:6801/2008222090","active_change":"2024-04-26T04:59:26.078964+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[{"gid":25099,"name":"x","mgr_features":4540701547738038271,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.1","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.1","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.12:8443/","prometheus":"http://172.21.15.12:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":80,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.12:0","nonce":1942073947}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.12:0","nonce":48739574}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.12:0","nonce":3209167518}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.12:0","nonce":61139418}]}]} 2024-04-26T05:05:02.735 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-26T05:05:02.735 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-26T05:05:02.736 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T05:05:03.000 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:04.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:04 smithi012 ceph-mon[91852]: pgmap v211: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:04.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:04 smithi012 ceph-mon[95978]: pgmap v211: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:04.813 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:05:04.814 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":100,"fsid":"0b9350c6-0388-11ef-bc93-c7b262605968","created":"2024-04-26T04:48:17.966774+0000","modified":"2024-04-26T05:04:47.495683+0000","last_up_change":"2024-04-26T05:04:46.493836+0000","last_in_change":"2024-04-26T04:53:04.660871+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:51:27.066997+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-26T04:53:56.453197+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-26T04:55:06.383315+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-26T04:55:06.882235+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"5f43d5c7-23d9-48a5-bc3c-a3a6ce3c004f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":80,"up_from":84,"up_thru":99,"down_at":81,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6802","nonce":1787752712},{"type":"v1","addr":"172.21.15.12:6803","nonce":1787752712}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6804","nonce":1787752712},{"type":"v1","addr":"172.21.15.12:6805","nonce":1787752712}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6808","nonce":1787752712},{"type":"v1","addr":"172.21.15.12:6809","nonce":1787752712}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6806","nonce":1787752712},{"type":"v1","addr":"172.21.15.12:6807","nonce":1787752712}]},"public_addr":"172.21.15.12:6803/1787752712","cluster_addr":"172.21.15.12:6805/1787752712","heartbeat_back_addr":"172.21.15.12:6809/1787752712","heartbeat_front_addr":"172.21.15.12:6807/1787752712","state":["exists","up"]},{"osd":1,"uuid":"75471714-6e29-4e02-9c03-86ce919b1ff0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":85,"up_from":89,"up_thru":99,"down_at":86,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6810","nonce":990497535},{"type":"v1","addr":"172.21.15.12:6811","nonce":990497535}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6812","nonce":990497535},{"type":"v1","addr":"172.21.15.12:6813","nonce":990497535}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6816","nonce":990497535},{"type":"v1","addr":"172.21.15.12:6817","nonce":990497535}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6814","nonce":990497535},{"type":"v1","addr":"172.21.15.12:6815","nonce":990497535}]},"public_addr":"172.21.15.12:6811/990497535","cluster_addr":"172.21.15.12:6813/990497535","heartbeat_back_addr":"172.21.15.12:6817/990497535","heartbeat_front_addr":"172.21.15.12:6815/990497535","state":["exists","up"]},{"osd":2,"uuid":"4b402201-4f28-4bc1-9cf0-0384090fb158","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":90,"up_from":94,"up_thru":99,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6818","nonce":2306607800},{"type":"v1","addr":"172.21.15.12:6819","nonce":2306607800}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6820","nonce":2306607800},{"type":"v1","addr":"172.21.15.12:6821","nonce":2306607800}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6824","nonce":2306607800},{"type":"v1","addr":"172.21.15.12:6825","nonce":2306607800}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6822","nonce":2306607800},{"type":"v1","addr":"172.21.15.12:6823","nonce":2306607800}]},"public_addr":"172.21.15.12:6819/2306607800","cluster_addr":"172.21.15.12:6821/2306607800","heartbeat_back_addr":"172.21.15.12:6825/2306607800","heartbeat_front_addr":"172.21.15.12:6823/2306607800","state":["exists","up"]},{"osd":3,"uuid":"54bf03b9-b2e6-4230-8dcd-d62af8853c36","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":95,"up_from":99,"up_thru":99,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6826","nonce":38281816},{"type":"v1","addr":"172.21.15.12:6827","nonce":38281816}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6828","nonce":38281816},{"type":"v1","addr":"172.21.15.12:6829","nonce":38281816}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6832","nonce":38281816},{"type":"v1","addr":"172.21.15.12:6833","nonce":38281816}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6830","nonce":38281816},{"type":"v1","addr":"172.21.15.12:6831","nonce":38281816}]},"public_addr":"172.21.15.12:6827/38281816","cluster_addr":"172.21.15.12:6829/38281816","heartbeat_back_addr":"172.21.15.12:6833/38281816","heartbeat_front_addr":"172.21.15.12:6831/38281816","state":["exists","up"]},{"osd":4,"uuid":"ea9d3fb5-1877-4971-a8fb-8b5aafa7871a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":99,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6800","nonce":2011782592},{"type":"v1","addr":"172.21.15.116:6801","nonce":2011782592}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6802","nonce":2011782592},{"type":"v1","addr":"172.21.15.116:6803","nonce":2011782592}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6806","nonce":2011782592},{"type":"v1","addr":"172.21.15.116:6807","nonce":2011782592}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6804","nonce":2011782592},{"type":"v1","addr":"172.21.15.116:6805","nonce":2011782592}]},"public_addr":"172.21.15.116:6801/2011782592","cluster_addr":"172.21.15.116:6803/2011782592","heartbeat_back_addr":"172.21.15.116:6807/2011782592","heartbeat_front_addr":"172.21.15.116:6805/2011782592","state":["exists","up"]},{"osd":5,"uuid":"59f0530d-37eb-41d8-86d0-580ba9b8e4e2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":99,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6808","nonce":4077435109},{"type":"v1","addr":"172.21.15.116:6809","nonce":4077435109}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6810","nonce":4077435109},{"type":"v1","addr":"172.21.15.116:6811","nonce":4077435109}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6814","nonce":4077435109},{"type":"v1","addr":"172.21.15.116:6815","nonce":4077435109}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6812","nonce":4077435109},{"type":"v1","addr":"172.21.15.116:6813","nonce":4077435109}]},"public_addr":"172.21.15.116:6809/4077435109","cluster_addr":"172.21.15.116:6811/4077435109","heartbeat_back_addr":"172.21.15.116:6815/4077435109","heartbeat_front_addr":"172.21.15.116:6813/4077435109","state":["exists","up"]},{"osd":6,"uuid":"62b1edf5-75b9-4ea6-af34-d40584a6c343","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":96,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6816","nonce":2560073849},{"type":"v1","addr":"172.21.15.116:6817","nonce":2560073849}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6818","nonce":2560073849},{"type":"v1","addr":"172.21.15.116:6819","nonce":2560073849}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6822","nonce":2560073849},{"type":"v1","addr":"172.21.15.116:6823","nonce":2560073849}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6820","nonce":2560073849},{"type":"v1","addr":"172.21.15.116:6821","nonce":2560073849}]},"public_addr":"172.21.15.116:6817/2560073849","cluster_addr":"172.21.15.116:6819/2560073849","heartbeat_back_addr":"172.21.15.116:6823/2560073849","heartbeat_front_addr":"172.21.15.116:6821/2560073849","state":["exists","up"]},{"osd":7,"uuid":"439d336e-0012-4b61-893f-50f562863965","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":99,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6824","nonce":3779511167},{"type":"v1","addr":"172.21.15.116:6825","nonce":3779511167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6826","nonce":3779511167},{"type":"v1","addr":"172.21.15.116:6827","nonce":3779511167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6830","nonce":3779511167},{"type":"v1","addr":"172.21.15.116:6831","nonce":3779511167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6828","nonce":3779511167},{"type":"v1","addr":"172.21.15.116:6829","nonce":3779511167}]},"public_addr":"172.21.15.116:6825/3779511167","cluster_addr":"172.21.15.116:6827/3779511167","heartbeat_back_addr":"172.21.15.116:6831/3779511167","heartbeat_front_addr":"172.21.15.116:6829/3779511167","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:50:34.985771+0000","dead_epoch":80},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:50:58.677899+0000","dead_epoch":85},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:51:23.046413+0000","dead_epoch":90},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:51:48.496821+0000","dead_epoch":95},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:09.812658+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:32.853940+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:56.215123+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:53:19.840326+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.12","mappings":[{"from":1,"to":5}]},{"pgid":"4.17","mappings":[{"from":1,"to":2}]},{"pgid":"4.1b","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.12:0/3390053105":"2024-04-27T04:57:35.286932+0000","172.21.15.12:0/1150032450":"2024-04-27T04:49:16.560212+0000","172.21.15.116:0/231391337":"2024-04-27T04:58:40.574973+0000","172.21.15.116:6834/235380242":"2024-04-27T04:58:40.574973+0000","172.21.15.12:6800/378740862":"2024-04-27T04:48:52.473898+0000","172.21.15.12:6800/2470533793":"2024-04-27T04:57:35.286932+0000","172.21.15.12:0/4212767587":"2024-04-27T04:49:16.560212+0000","172.21.15.116:0/1470699755":"2024-04-27T04:58:40.574973+0000","172.21.15.12:6801/1235528000":"2024-04-27T04:53:52.394847+0000","172.21.15.12:0/3564895536":"2024-04-27T04:59:26.078714+0000","172.21.15.116:0/3251959094":"2024-04-27T04:58:40.574973+0000","172.21.15.12:6800/2681852455":"2024-04-27T04:49:16.560212+0000","172.21.15.12:0/1826681709":"2024-04-27T04:57:35.286932+0000","172.21.15.12:0/320935135":"2024-04-27T04:48:52.473898+0000","172.21.15.12:0/743849710":"2024-04-27T04:59:26.078714+0000","172.21.15.12:0/2421619470":"2024-04-27T04:53:52.394847+0000","172.21.15.12:6801/2681852455":"2024-04-27T04:49:16.560212+0000","172.21.15.12:0/2851438792":"2024-04-27T04:59:26.078714+0000","172.21.15.116:6835/235380242":"2024-04-27T04:58:40.574973+0000","172.21.15.12:0/625629550":"2024-04-27T04:53:52.394847+0000","172.21.15.116:0/4243638523":"2024-04-27T04:58:40.574973+0000","172.21.15.12:6801/2470533793":"2024-04-27T04:57:35.286932+0000","172.21.15.12:0/1087977851":"2024-04-27T04:53:52.394847+0000","172.21.15.116:0/3570968272":"2024-04-27T04:58:40.574973+0000","172.21.15.12:6801/378740862":"2024-04-27T04:48:52.473898+0000","172.21.15.116:0/3667542398":"2024-04-27T04:58:40.574973+0000","172.21.15.12:0/824190310":"2024-04-27T04:53:52.394847+0000","172.21.15.12:0/1805372288":"2024-04-27T04:59:26.078714+0000","172.21.15.12:6800/1235528000":"2024-04-27T04:53:52.394847+0000","172.21.15.12:0/2567163031":"2024-04-27T04:57:35.286932+0000","172.21.15.12:0/2016967349":"2024-04-27T04:48:52.473898+0000","172.21.15.12:0/4007370348":"2024-04-27T04:49:16.560212+0000","172.21.15.12:0/3076748047":"2024-04-27T04:59:26.078714+0000","172.21.15.12:6800/1044650693":"2024-04-27T04:59:26.078714+0000","172.21.15.12:6801/1044650693":"2024-04-27T04:59:26.078714+0000","172.21.15.12:0/3389110023":"2024-04-27T04:59:26.078714+0000","172.21.15.12:0/2417119692":"2024-04-27T04:57:35.286932+0000","172.21.15.12:0/2418603751":"2024-04-27T04:48:52.473898+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T05:05:04.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:04 smithi116 ceph-mon[85502]: pgmap v211: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:05.389 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-26T05:05:05.390 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T05:05:05.656 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:05.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:05 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/3759542431' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T05:05:05.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:05 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/3759542431' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T05:05:05.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:05 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/3759542431' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T05:05:06.626 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:06 smithi012 ceph-mon[91852]: pgmap v212: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:06.626 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:06 smithi012 ceph-mon[95978]: pgmap v212: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:06.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:06 smithi116 ceph-mon[85502]: pgmap v212: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:07.351 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:05:07.351 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":100,"fsid":"0b9350c6-0388-11ef-bc93-c7b262605968","created":"2024-04-26T04:48:17.966774+0000","modified":"2024-04-26T05:04:47.495683+0000","last_up_change":"2024-04-26T05:04:46.493836+0000","last_in_change":"2024-04-26T04:53:04.660871+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:51:27.066997+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-26T04:53:56.453197+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-26T04:55:06.383315+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-26T04:55:06.882235+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"5f43d5c7-23d9-48a5-bc3c-a3a6ce3c004f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":80,"up_from":84,"up_thru":99,"down_at":81,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6802","nonce":1787752712},{"type":"v1","addr":"172.21.15.12:6803","nonce":1787752712}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6804","nonce":1787752712},{"type":"v1","addr":"172.21.15.12:6805","nonce":1787752712}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6808","nonce":1787752712},{"type":"v1","addr":"172.21.15.12:6809","nonce":1787752712}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6806","nonce":1787752712},{"type":"v1","addr":"172.21.15.12:6807","nonce":1787752712}]},"public_addr":"172.21.15.12:6803/1787752712","cluster_addr":"172.21.15.12:6805/1787752712","heartbeat_back_addr":"172.21.15.12:6809/1787752712","heartbeat_front_addr":"172.21.15.12:6807/1787752712","state":["exists","up"]},{"osd":1,"uuid":"75471714-6e29-4e02-9c03-86ce919b1ff0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":85,"up_from":89,"up_thru":99,"down_at":86,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6810","nonce":990497535},{"type":"v1","addr":"172.21.15.12:6811","nonce":990497535}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6812","nonce":990497535},{"type":"v1","addr":"172.21.15.12:6813","nonce":990497535}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6816","nonce":990497535},{"type":"v1","addr":"172.21.15.12:6817","nonce":990497535}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6814","nonce":990497535},{"type":"v1","addr":"172.21.15.12:6815","nonce":990497535}]},"public_addr":"172.21.15.12:6811/990497535","cluster_addr":"172.21.15.12:6813/990497535","heartbeat_back_addr":"172.21.15.12:6817/990497535","heartbeat_front_addr":"172.21.15.12:6815/990497535","state":["exists","up"]},{"osd":2,"uuid":"4b402201-4f28-4bc1-9cf0-0384090fb158","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":90,"up_from":94,"up_thru":99,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6818","nonce":2306607800},{"type":"v1","addr":"172.21.15.12:6819","nonce":2306607800}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6820","nonce":2306607800},{"type":"v1","addr":"172.21.15.12:6821","nonce":2306607800}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6824","nonce":2306607800},{"type":"v1","addr":"172.21.15.12:6825","nonce":2306607800}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6822","nonce":2306607800},{"type":"v1","addr":"172.21.15.12:6823","nonce":2306607800}]},"public_addr":"172.21.15.12:6819/2306607800","cluster_addr":"172.21.15.12:6821/2306607800","heartbeat_back_addr":"172.21.15.12:6825/2306607800","heartbeat_front_addr":"172.21.15.12:6823/2306607800","state":["exists","up"]},{"osd":3,"uuid":"54bf03b9-b2e6-4230-8dcd-d62af8853c36","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":95,"up_from":99,"up_thru":99,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6826","nonce":38281816},{"type":"v1","addr":"172.21.15.12:6827","nonce":38281816}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6828","nonce":38281816},{"type":"v1","addr":"172.21.15.12:6829","nonce":38281816}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6832","nonce":38281816},{"type":"v1","addr":"172.21.15.12:6833","nonce":38281816}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6830","nonce":38281816},{"type":"v1","addr":"172.21.15.12:6831","nonce":38281816}]},"public_addr":"172.21.15.12:6827/38281816","cluster_addr":"172.21.15.12:6829/38281816","heartbeat_back_addr":"172.21.15.12:6833/38281816","heartbeat_front_addr":"172.21.15.12:6831/38281816","state":["exists","up"]},{"osd":4,"uuid":"ea9d3fb5-1877-4971-a8fb-8b5aafa7871a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":99,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6800","nonce":2011782592},{"type":"v1","addr":"172.21.15.116:6801","nonce":2011782592}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6802","nonce":2011782592},{"type":"v1","addr":"172.21.15.116:6803","nonce":2011782592}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6806","nonce":2011782592},{"type":"v1","addr":"172.21.15.116:6807","nonce":2011782592}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6804","nonce":2011782592},{"type":"v1","addr":"172.21.15.116:6805","nonce":2011782592}]},"public_addr":"172.21.15.116:6801/2011782592","cluster_addr":"172.21.15.116:6803/2011782592","heartbeat_back_addr":"172.21.15.116:6807/2011782592","heartbeat_front_addr":"172.21.15.116:6805/2011782592","state":["exists","up"]},{"osd":5,"uuid":"59f0530d-37eb-41d8-86d0-580ba9b8e4e2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":99,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6808","nonce":4077435109},{"type":"v1","addr":"172.21.15.116:6809","nonce":4077435109}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6810","nonce":4077435109},{"type":"v1","addr":"172.21.15.116:6811","nonce":4077435109}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6814","nonce":4077435109},{"type":"v1","addr":"172.21.15.116:6815","nonce":4077435109}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6812","nonce":4077435109},{"type":"v1","addr":"172.21.15.116:6813","nonce":4077435109}]},"public_addr":"172.21.15.116:6809/4077435109","cluster_addr":"172.21.15.116:6811/4077435109","heartbeat_back_addr":"172.21.15.116:6815/4077435109","heartbeat_front_addr":"172.21.15.116:6813/4077435109","state":["exists","up"]},{"osd":6,"uuid":"62b1edf5-75b9-4ea6-af34-d40584a6c343","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":96,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6816","nonce":2560073849},{"type":"v1","addr":"172.21.15.116:6817","nonce":2560073849}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6818","nonce":2560073849},{"type":"v1","addr":"172.21.15.116:6819","nonce":2560073849}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6822","nonce":2560073849},{"type":"v1","addr":"172.21.15.116:6823","nonce":2560073849}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6820","nonce":2560073849},{"type":"v1","addr":"172.21.15.116:6821","nonce":2560073849}]},"public_addr":"172.21.15.116:6817/2560073849","cluster_addr":"172.21.15.116:6819/2560073849","heartbeat_back_addr":"172.21.15.116:6823/2560073849","heartbeat_front_addr":"172.21.15.116:6821/2560073849","state":["exists","up"]},{"osd":7,"uuid":"439d336e-0012-4b61-893f-50f562863965","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":99,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6824","nonce":3779511167},{"type":"v1","addr":"172.21.15.116:6825","nonce":3779511167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6826","nonce":3779511167},{"type":"v1","addr":"172.21.15.116:6827","nonce":3779511167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6830","nonce":3779511167},{"type":"v1","addr":"172.21.15.116:6831","nonce":3779511167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6828","nonce":3779511167},{"type":"v1","addr":"172.21.15.116:6829","nonce":3779511167}]},"public_addr":"172.21.15.116:6825/3779511167","cluster_addr":"172.21.15.116:6827/3779511167","heartbeat_back_addr":"172.21.15.116:6831/3779511167","heartbeat_front_addr":"172.21.15.116:6829/3779511167","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:50:34.985771+0000","dead_epoch":80},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:50:58.677899+0000","dead_epoch":85},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:51:23.046413+0000","dead_epoch":90},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:51:48.496821+0000","dead_epoch":95},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:09.812658+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:32.853940+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:52:56.215123+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:53:19.840326+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.12","mappings":[{"from":1,"to":5}]},{"pgid":"4.17","mappings":[{"from":1,"to":2}]},{"pgid":"4.1b","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.12:0/3390053105":"2024-04-27T04:57:35.286932+0000","172.21.15.12:0/1150032450":"2024-04-27T04:49:16.560212+0000","172.21.15.116:0/231391337":"2024-04-27T04:58:40.574973+0000","172.21.15.116:6834/235380242":"2024-04-27T04:58:40.574973+0000","172.21.15.12:6800/378740862":"2024-04-27T04:48:52.473898+0000","172.21.15.12:6800/2470533793":"2024-04-27T04:57:35.286932+0000","172.21.15.12:0/4212767587":"2024-04-27T04:49:16.560212+0000","172.21.15.116:0/1470699755":"2024-04-27T04:58:40.574973+0000","172.21.15.12:6801/1235528000":"2024-04-27T04:53:52.394847+0000","172.21.15.12:0/3564895536":"2024-04-27T04:59:26.078714+0000","172.21.15.116:0/3251959094":"2024-04-27T04:58:40.574973+0000","172.21.15.12:6800/2681852455":"2024-04-27T04:49:16.560212+0000","172.21.15.12:0/1826681709":"2024-04-27T04:57:35.286932+0000","172.21.15.12:0/320935135":"2024-04-27T04:48:52.473898+0000","172.21.15.12:0/743849710":"2024-04-27T04:59:26.078714+0000","172.21.15.12:0/2421619470":"2024-04-27T04:53:52.394847+0000","172.21.15.12:6801/2681852455":"2024-04-27T04:49:16.560212+0000","172.21.15.12:0/2851438792":"2024-04-27T04:59:26.078714+0000","172.21.15.116:6835/235380242":"2024-04-27T04:58:40.574973+0000","172.21.15.12:0/625629550":"2024-04-27T04:53:52.394847+0000","172.21.15.116:0/4243638523":"2024-04-27T04:58:40.574973+0000","172.21.15.12:6801/2470533793":"2024-04-27T04:57:35.286932+0000","172.21.15.12:0/1087977851":"2024-04-27T04:53:52.394847+0000","172.21.15.116:0/3570968272":"2024-04-27T04:58:40.574973+0000","172.21.15.12:6801/378740862":"2024-04-27T04:48:52.473898+0000","172.21.15.116:0/3667542398":"2024-04-27T04:58:40.574973+0000","172.21.15.12:0/824190310":"2024-04-27T04:53:52.394847+0000","172.21.15.12:0/1805372288":"2024-04-27T04:59:26.078714+0000","172.21.15.12:6800/1235528000":"2024-04-27T04:53:52.394847+0000","172.21.15.12:0/2567163031":"2024-04-27T04:57:35.286932+0000","172.21.15.12:0/2016967349":"2024-04-27T04:48:52.473898+0000","172.21.15.12:0/4007370348":"2024-04-27T04:49:16.560212+0000","172.21.15.12:0/3076748047":"2024-04-27T04:59:26.078714+0000","172.21.15.12:6800/1044650693":"2024-04-27T04:59:26.078714+0000","172.21.15.12:6801/1044650693":"2024-04-27T04:59:26.078714+0000","172.21.15.12:0/3389110023":"2024-04-27T04:59:26.078714+0000","172.21.15.12:0/2417119692":"2024-04-27T04:57:35.286932+0000","172.21.15.12:0/2418603751":"2024-04-27T04:48:52.473898+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T05:05:07.976 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:07 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/87715640' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T05:05:07.976 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:07 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/87715640' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T05:05:07.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:07 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/87715640' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T05:05:08.008 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-26T05:05:08.009 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-26T05:05:08.009 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-26T05:05:08.009 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-26T05:05:08.010 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-26T05:05:08.010 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-26T05:05:08.011 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-26T05:05:08.011 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-26T05:05:08.264 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:05:07 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:05:07] "GET /metrics HTTP/1.1" 200 34020 "" "Prometheus/2.43.0" 2024-04-26T05:05:08.604 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:08.604 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:08.605 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:08.606 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:08.609 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:08.615 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:08.620 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:08.660 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:08.751 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:08 smithi012 ceph-mon[91852]: pgmap v213: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:08.751 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:08 smithi012 ceph-mon[95978]: pgmap v213: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:08.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:08 smithi116 ceph-mon[85502]: pgmap v213: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:10.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:10 smithi012 ceph-mon[91852]: pgmap v214: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:10.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:10 smithi012 ceph-mon[95978]: pgmap v214: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:10.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:10 smithi116 ceph-mon[85502]: pgmap v214: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:11.746 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:11 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:11.746 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:11 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:11.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:11 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:12.974 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:12 smithi012 ceph-mon[91852]: pgmap v215: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:12.974 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:12 smithi012 ceph-mon[95978]: pgmap v215: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:12.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:12 smithi116 ceph-mon[85502]: pgmap v215: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:14.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:14 smithi012 ceph-mon[91852]: pgmap v216: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:14.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:14 smithi012 ceph-mon[95978]: pgmap v216: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:14.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:14 smithi116 ceph-mon[85502]: pgmap v216: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:16.863 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:16 smithi012 ceph-mon[91852]: pgmap v217: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:16.864 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:16 smithi012 ceph-mon[95978]: pgmap v217: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:16.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:16 smithi116 ceph-mon[85502]: pgmap v217: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:18.080 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:05:17 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:05:17] "GET /metrics HTTP/1.1" 200 34022 "" "Prometheus/2.43.0" 2024-04-26T05:05:18.703 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:18 smithi012 ceph-mon[91852]: pgmap v218: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:18.703 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:18 smithi012 ceph-mon[95978]: pgmap v218: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:18.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:18 smithi116 ceph-mon[85502]: pgmap v218: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:19.912 INFO:teuthology.orchestra.run.smithi012.stdout:133143986335 2024-04-26T05:05:19.913 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-26T05:05:20.820 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:20 smithi012 ceph-mon[91852]: pgmap v219: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:20.821 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:20 smithi012 ceph-mon[95978]: pgmap v219: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:20.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:20 smithi116 ceph-mon[85502]: pgmap v219: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:21.770 INFO:teuthology.orchestra.run.smithi012.stdout:425201762313 2024-04-26T05:05:21.770 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-26T05:05:22.126 INFO:teuthology.orchestra.run.smithi012.stdout:382252089363 2024-04-26T05:05:22.127 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-26T05:05:22.644 INFO:teuthology.orchestra.run.smithi012.stdout:167503724700 2024-04-26T05:05:22.644 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-26T05:05:22.825 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:22 smithi012 ceph-mon[91852]: pgmap v220: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:22.825 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:22 smithi012 ceph-mon[95978]: pgmap v220: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:22.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:22 smithi116 ceph-mon[85502]: pgmap v220: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:23.300 INFO:teuthology.orchestra.run.smithi012.stdout:403726925839 2024-04-26T05:05:23.301 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-26T05:05:23.844 INFO:teuthology.orchestra.run.smithi012.stdout:201863463063 2024-04-26T05:05:23.844 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-26T05:05:24.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:24 smithi012 ceph-mon[95978]: pgmap v221: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:24.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:24 smithi012 ceph-mon[91852]: pgmap v221: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:24.836 INFO:teuthology.orchestra.run.smithi012.stdout:360777252898 2024-04-26T05:05:24.837 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-26T05:05:24.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:24 smithi116 ceph-mon[85502]: pgmap v221: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:25.197 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:25.197 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:25.197 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:25.198 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:25.235 INFO:teuthology.orchestra.run.smithi012.stdout:236223201427 2024-04-26T05:05:25.235 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-26T05:05:25.285 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:26.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:26 smithi012 ceph-mon[91852]: pgmap v222: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:26.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:26 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:26.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:26 smithi012 ceph-mon[95978]: pgmap v222: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:26.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:26 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:26.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:26 smithi116 ceph-mon[85502]: pgmap v222: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:26.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:26 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:28.078 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:05:27 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:05:27] "GET /metrics HTTP/1.1" 200 34020 "" "Prometheus/2.43.0" 2024-04-26T05:05:28.118 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:28.118 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:28.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:28 smithi012 ceph-mon[91852]: pgmap v223: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:28.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:28 smithi012 ceph-mon[95978]: pgmap v223: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:28.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:28 smithi116 ceph-mon[85502]: pgmap v223: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:30.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:30 smithi012 ceph-mon[91852]: pgmap v224: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:30.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:30 smithi012 ceph-mon[95978]: pgmap v224: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:30.823 INFO:teuthology.orchestra.run.smithi012.stdout:382252089365 2024-04-26T05:05:30.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:30 smithi116 ceph-mon[85502]: pgmap v224: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:31.586 INFO:teuthology.orchestra.run.smithi012.stdout:403726925841 2024-04-26T05:05:31.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:31 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/3561713112' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T05:05:31.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:31 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/3801305182' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T05:05:31.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:31 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/3561713112' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T05:05:31.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:31 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/3801305182' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T05:05:31.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:31 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/3561713112' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T05:05:31.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:31 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/3801305182' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T05:05:32.750 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:32 smithi012 ceph-mon[91852]: pgmap v225: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:32.750 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:32 smithi012 ceph-mon[95978]: pgmap v225: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:32.755 INFO:teuthology.orchestra.run.smithi012.stdout:167503724702 2024-04-26T05:05:32.837 INFO:teuthology.orchestra.run.smithi012.stdout:133143986338 2024-04-26T05:05:32.893 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:32 smithi116 ceph-mon[85502]: pgmap v225: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:33.151 INFO:tasks.cephadm.ceph_manager.ceph:need seq 403726925839 got 403726925841 for osd.2 2024-04-26T05:05:33.151 DEBUG:teuthology.parallel:result is None 2024-04-26T05:05:33.705 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:33 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/3013446333' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T05:05:33.706 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:33 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/2924559506' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T05:05:33.706 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:33 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/3013446333' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T05:05:33.706 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:33 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/2924559506' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T05:05:33.737 INFO:tasks.cephadm.ceph_manager.ceph:need seq 382252089363 got 382252089365 for osd.1 2024-04-26T05:05:33.737 DEBUG:teuthology.parallel:result is None 2024-04-26T05:05:33.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:33 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/3013446333' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T05:05:33.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:33 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/2924559506' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T05:05:34.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:34 smithi012 ceph-mon[91852]: pgmap v226: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:34.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:34 smithi012 ceph-mon[95978]: pgmap v226: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:34.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:34 smithi116 ceph-mon[85502]: pgmap v226: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:35.428 INFO:teuthology.orchestra.run.smithi012.stdout:360777252900 2024-04-26T05:05:35.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:35 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/3458494091' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T05:05:35.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:35 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/3458494091' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T05:05:35.814 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724700 got 167503724702 for osd.5 2024-04-26T05:05:35.814 DEBUG:teuthology.parallel:result is None 2024-04-26T05:05:35.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:35 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/3458494091' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T05:05:36.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:36 smithi012 ceph-mon[91852]: pgmap v227: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:36.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:36 smithi012 ceph-mon[95978]: pgmap v227: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:36.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:36 smithi116 ceph-mon[85502]: pgmap v227: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:37.350 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:37.378 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986335 got 133143986338 for osd.4 2024-04-26T05:05:37.378 DEBUG:teuthology.parallel:result is None 2024-04-26T05:05:37.693 INFO:tasks.cephadm.ceph_manager.ceph:need seq 360777252898 got 360777252900 for osd.0 2024-04-26T05:05:37.693 DEBUG:teuthology.parallel:result is None 2024-04-26T05:05:37.788 INFO:teuthology.orchestra.run.smithi012.stdout:425201762316 2024-04-26T05:05:38.199 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:05:37 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:05:37] "GET /metrics HTTP/1.1" 200 34020 "" "Prometheus/2.43.0" 2024-04-26T05:05:38.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:38 smithi012 ceph-mon[91852]: pgmap v228: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:38.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:38 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/2556208807' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T05:05:38.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:38 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/682407699' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T05:05:38.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:38 smithi012 ceph-mon[95978]: pgmap v228: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:38.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:38 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/2556208807' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T05:05:38.765 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:38 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/682407699' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T05:05:38.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:38 smithi116 ceph-mon[85502]: pgmap v228: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:38.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:38 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/2556208807' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T05:05:38.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:38 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/682407699' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T05:05:39.234 INFO:tasks.cephadm.ceph_manager.ceph:need seq 425201762313 got 425201762316 for osd.3 2024-04-26T05:05:39.234 DEBUG:teuthology.parallel:result is None 2024-04-26T05:05:39.331 INFO:teuthology.orchestra.run.smithi012.stdout:201863463067 2024-04-26T05:05:40.011 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863463063 got 201863463067 for osd.6 2024-04-26T05:05:40.012 DEBUG:teuthology.parallel:result is None 2024-04-26T05:05:40.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:40 smithi012 ceph-mon[91852]: pgmap v229: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:40.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:40 smithi012 ceph-mon[95978]: pgmap v229: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:40.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:40 smithi116 ceph-mon[85502]: pgmap v229: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:41.233 INFO:teuthology.orchestra.run.smithi012.stdout:236223201430 2024-04-26T05:05:41.940 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:41 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:41.940 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:41 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/1662850007' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T05:05:41.940 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:41 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:41.941 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:41 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/1662850007' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T05:05:41.972 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201427 got 236223201430 for osd.7 2024-04-26T05:05:41.972 DEBUG:teuthology.parallel:result is None 2024-04-26T05:05:41.972 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-26T05:05:41.972 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-26T05:05:41.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:41 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:41.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:41 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/1662850007' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T05:05:42.247 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:42.668 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:42 smithi012 ceph-mon[91852]: pgmap v230: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:42.669 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:42 smithi012 ceph-mon[95978]: pgmap v230: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:42.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:42 smithi116 ceph-mon[85502]: pgmap v230: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:44.258 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:05:44.259 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2024-04-26T05:05:44.881 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:44 smithi012 ceph-mon[91852]: pgmap v231: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:44.881 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:44 smithi012 ceph-mon[95978]: pgmap v231: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:44.912 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":231,"stamp":"2024-04-26T05:05:43.470844+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595898,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1334444,"kb_used_data":10296,"kb_used_omap":89,"kb_used_meta":1323942,"kb_avail":748626772,"statfs":{"total":767960285184,"available":766593814528,"internally_reserved":0,"allocated":10543104,"data_stored":6236317,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":91664,"internal_metadata":1355717104},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002769"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.501003+0000","last_change":"2024-04-26T05:03:51.802774+0000","last_active":"2024-04-26T05:04:46.501003+0000","last_peered":"2024-04-26T05:04:46.501003+0000","last_clean":"2024-04-26T05:04:46.501003+0000","last_became_active":"2024-04-26T05:03:51.802166+0000","last_became_peered":"2024-04-26T05:03:51.802166+0000","last_unstale":"2024-04-26T05:04:46.501003+0000","last_undegraded":"2024-04-26T05:04:46.501003+0000","last_fullsized":"2024-04-26T05:04:46.501003+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:19:02.626403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":66,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.502100+0000","last_change":"2024-04-26T05:04:12.124459+0000","last_active":"2024-04-26T05:04:47.502100+0000","last_peered":"2024-04-26T05:04:47.502100+0000","last_clean":"2024-04-26T05:04:47.502100+0000","last_became_active":"2024-04-26T05:04:12.123993+0000","last_became_peered":"2024-04-26T05:04:12.123993+0000","last_unstale":"2024-04-26T05:04:47.502100+0000","last_undegraded":"2024-04-26T05:04:47.502100+0000","last_fullsized":"2024-04-26T05:04:47.502100+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:50:29.431399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":83,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.513958+0000","last_change":"2024-04-26T05:04:47.513958+0000","last_active":"2024-04-26T05:04:47.513958+0000","last_peered":"2024-04-26T05:04:47.513958+0000","last_clean":"2024-04-26T05:04:47.513958+0000","last_became_active":"2024-04-26T05:04:47.513425+0000","last_became_peered":"2024-04-26T05:04:47.513425+0000","last_unstale":"2024-04-26T05:04:47.513958+0000","last_undegraded":"2024-04-26T05:04:47.513958+0000","last_fullsized":"2024-04-26T05:04:47.513958+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:12:41.169186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":68,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.522311+0000","last_change":"2024-04-26T05:04:47.522311+0000","last_active":"2024-04-26T05:04:47.522311+0000","last_peered":"2024-04-26T05:04:47.522311+0000","last_clean":"2024-04-26T05:04:47.522311+0000","last_became_active":"2024-04-26T05:04:47.521599+0000","last_became_peered":"2024-04-26T05:04:47.521599+0000","last_unstale":"2024-04-26T05:04:47.522311+0000","last_undegraded":"2024-04-26T05:04:47.522311+0000","last_fullsized":"2024-04-26T05:04:47.522311+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:32:57.457648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":76,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.509750+0000","last_change":"2024-04-26T05:04:47.509750+0000","last_active":"2024-04-26T05:04:47.509750+0000","last_peered":"2024-04-26T05:04:47.509750+0000","last_clean":"2024-04-26T05:04:47.509750+0000","last_became_active":"2024-04-26T05:04:47.509377+0000","last_became_peered":"2024-04-26T05:04:47.509377+0000","last_unstale":"2024-04-26T05:04:47.509750+0000","last_undegraded":"2024-04-26T05:04:47.509750+0000","last_fullsized":"2024-04-26T05:04:47.509750+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:05:14.524534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":45,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.501340+0000","last_change":"2024-04-26T04:55:07.445793+0000","last_active":"2024-04-26T05:04:46.501340+0000","last_peered":"2024-04-26T05:04:46.501340+0000","last_clean":"2024-04-26T05:04:46.501340+0000","last_became_active":"2024-04-26T04:55:07.445496+0000","last_became_peered":"2024-04-26T04:55:07.445496+0000","last_unstale":"2024-04-26T05:04:46.501340+0000","last_undegraded":"2024-04-26T05:04:46.501340+0000","last_fullsized":"2024-04-26T05:04:46.501340+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:53:56.081693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":86,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.509867+0000","last_change":"2024-04-26T05:04:47.509867+0000","last_active":"2024-04-26T05:04:47.509867+0000","last_peered":"2024-04-26T05:04:47.509867+0000","last_clean":"2024-04-26T05:04:47.509867+0000","last_became_active":"2024-04-26T05:04:47.509500+0000","last_became_peered":"2024-04-26T05:04:47.509500+0000","last_unstale":"2024-04-26T05:04:47.509867+0000","last_undegraded":"2024-04-26T05:04:47.509867+0000","last_fullsized":"2024-04-26T05:04:47.509867+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:58:05.754843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":62,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.500044+0000","last_change":"2024-04-26T05:03:51.801296+0000","last_active":"2024-04-26T05:04:46.500044+0000","last_peered":"2024-04-26T05:04:46.500044+0000","last_clean":"2024-04-26T05:04:46.500044+0000","last_became_active":"2024-04-26T05:03:51.800904+0000","last_became_peered":"2024-04-26T05:03:51.800904+0000","last_unstale":"2024-04-26T05:04:46.500044+0000","last_undegraded":"2024-04-26T05:04:46.500044+0000","last_fullsized":"2024-04-26T05:04:46.500044+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:23:49.534793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":80,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.520013+0000","last_change":"2024-04-26T05:04:47.520013+0000","last_active":"2024-04-26T05:04:47.520013+0000","last_peered":"2024-04-26T05:04:47.520013+0000","last_clean":"2024-04-26T05:04:47.520013+0000","last_became_active":"2024-04-26T05:04:47.519664+0000","last_became_peered":"2024-04-26T05:04:47.519664+0000","last_unstale":"2024-04-26T05:04:47.520013+0000","last_undegraded":"2024-04-26T05:04:47.520013+0000","last_fullsized":"2024-04-26T05:04:47.520013+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:11:24.790879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":57,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.501904+0000","last_change":"2024-04-26T05:02:40.771411+0000","last_active":"2024-04-26T05:04:47.501904+0000","last_peered":"2024-04-26T05:04:47.501904+0000","last_clean":"2024-04-26T05:04:47.501904+0000","last_became_active":"2024-04-26T05:02:40.770506+0000","last_became_peered":"2024-04-26T05:02:40.770506+0000","last_unstale":"2024-04-26T05:04:47.501904+0000","last_undegraded":"2024-04-26T05:04:47.501904+0000","last_fullsized":"2024-04-26T05:04:47.501904+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:49:20.399132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":42,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.501215+0000","last_change":"2024-04-26T04:55:07.474089+0000","last_active":"2024-04-26T05:04:46.501215+0000","last_peered":"2024-04-26T05:04:46.501215+0000","last_clean":"2024-04-26T05:04:46.501215+0000","last_became_active":"2024-04-26T04:55:07.473624+0000","last_became_peered":"2024-04-26T04:55:07.473624+0000","last_unstale":"2024-04-26T05:04:46.501215+0000","last_undegraded":"2024-04-26T05:04:46.501215+0000","last_fullsized":"2024-04-26T05:04:46.501215+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:15:47.507462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":66,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.501978+0000","last_change":"2024-04-26T05:03:51.806178+0000","last_active":"2024-04-26T05:04:47.501978+0000","last_peered":"2024-04-26T05:04:47.501978+0000","last_clean":"2024-04-26T05:04:47.501978+0000","last_became_active":"2024-04-26T05:03:51.805803+0000","last_became_peered":"2024-04-26T05:03:51.805803+0000","last_unstale":"2024-04-26T05:04:47.501978+0000","last_undegraded":"2024-04-26T05:04:47.501978+0000","last_fullsized":"2024-04-26T05:04:47.501978+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:54:57.751726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":64,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.505722+0000","last_change":"2024-04-26T05:04:12.123626+0000","last_active":"2024-04-26T05:04:46.505722+0000","last_peered":"2024-04-26T05:04:46.505722+0000","last_clean":"2024-04-26T05:04:46.505722+0000","last_became_active":"2024-04-26T05:04:12.123268+0000","last_became_peered":"2024-04-26T05:04:12.123268+0000","last_unstale":"2024-04-26T05:04:46.505722+0000","last_undegraded":"2024-04-26T05:04:46.505722+0000","last_fullsized":"2024-04-26T05:04:46.505722+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:11:41.914165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":57,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.516569+0000","last_change":"2024-04-26T05:04:47.516569+0000","last_active":"2024-04-26T05:04:47.516569+0000","last_peered":"2024-04-26T05:04:47.516569+0000","last_clean":"2024-04-26T05:04:47.516569+0000","last_became_active":"2024-04-26T05:04:47.516062+0000","last_became_peered":"2024-04-26T05:04:47.516062+0000","last_unstale":"2024-04-26T05:04:47.516569+0000","last_undegraded":"2024-04-26T05:04:47.516569+0000","last_fullsized":"2024-04-26T05:04:47.516569+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:43:44.271116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":42,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.507556+0000","last_change":"2024-04-26T05:03:51.807810+0000","last_active":"2024-04-26T05:04:47.507556+0000","last_peered":"2024-04-26T05:04:47.507556+0000","last_clean":"2024-04-26T05:04:47.507556+0000","last_became_active":"2024-04-26T05:03:51.805285+0000","last_became_peered":"2024-04-26T05:03:51.805285+0000","last_unstale":"2024-04-26T05:04:47.507556+0000","last_undegraded":"2024-04-26T05:04:47.507556+0000","last_fullsized":"2024-04-26T05:04:47.507556+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:39:43.835533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":60,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.519997+0000","last_change":"2024-04-26T05:04:47.519997+0000","last_active":"2024-04-26T05:04:47.519997+0000","last_peered":"2024-04-26T05:04:47.519997+0000","last_clean":"2024-04-26T05:04:47.519997+0000","last_became_active":"2024-04-26T05:04:47.519403+0000","last_became_peered":"2024-04-26T05:04:47.519403+0000","last_unstale":"2024-04-26T05:04:47.519997+0000","last_undegraded":"2024-04-26T05:04:47.519997+0000","last_fullsized":"2024-04-26T05:04:47.519997+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:20:46.301349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":54,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.501680+0000","last_change":"2024-04-26T05:02:40.772049+0000","last_active":"2024-04-26T05:04:47.501680+0000","last_peered":"2024-04-26T05:04:47.501680+0000","last_clean":"2024-04-26T05:04:47.501680+0000","last_became_active":"2024-04-26T05:02:40.771105+0000","last_became_peered":"2024-04-26T05:02:40.771105+0000","last_unstale":"2024-04-26T05:04:47.501680+0000","last_undegraded":"2024-04-26T05:04:47.501680+0000","last_fullsized":"2024-04-26T05:04:47.501680+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:29:57.754882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":51,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.508294+0000","last_change":"2024-04-26T05:04:12.126757+0000","last_active":"2024-04-26T05:04:47.508294+0000","last_peered":"2024-04-26T05:04:47.508294+0000","last_clean":"2024-04-26T05:04:47.508294+0000","last_became_active":"2024-04-26T05:04:12.126449+0000","last_became_peered":"2024-04-26T05:04:12.126449+0000","last_unstale":"2024-04-26T05:04:47.508294+0000","last_undegraded":"2024-04-26T05:04:47.508294+0000","last_fullsized":"2024-04-26T05:04:47.508294+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:09:15.930209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":63,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.498093+0000","last_change":"2024-04-26T05:02:40.767283+0000","last_active":"2024-04-26T05:04:46.498093+0000","last_peered":"2024-04-26T05:04:46.498093+0000","last_clean":"2024-04-26T05:04:46.498093+0000","last_became_active":"2024-04-26T05:02:40.766809+0000","last_became_peered":"2024-04-26T05:02:40.766809+0000","last_unstale":"2024-04-26T05:04:46.498093+0000","last_undegraded":"2024-04-26T05:04:46.498093+0000","last_fullsized":"2024-04-26T05:04:46.498093+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:09:39.752228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":72,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.502081+0000","last_change":"2024-04-26T05:04:12.124689+0000","last_active":"2024-04-26T05:04:47.502081+0000","last_peered":"2024-04-26T05:04:47.502081+0000","last_clean":"2024-04-26T05:04:47.502081+0000","last_became_active":"2024-04-26T05:04:12.124354+0000","last_became_peered":"2024-04-26T05:04:12.124354+0000","last_unstale":"2024-04-26T05:04:47.502081+0000","last_undegraded":"2024-04-26T05:04:47.502081+0000","last_fullsized":"2024-04-26T05:04:47.502081+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:54:42.387778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":62,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.513467+0000","last_change":"2024-04-26T05:04:47.513467+0000","last_active":"2024-04-26T05:04:47.513467+0000","last_peered":"2024-04-26T05:04:47.513467+0000","last_clean":"2024-04-26T05:04:47.513467+0000","last_became_active":"2024-04-26T05:04:47.513152+0000","last_became_peered":"2024-04-26T05:04:47.513152+0000","last_unstale":"2024-04-26T05:04:47.513467+0000","last_undegraded":"2024-04-26T05:04:47.513467+0000","last_fullsized":"2024-04-26T05:04:47.513467+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:12:13.750681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":80,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.498769+0000","last_change":"2024-04-26T05:03:51.793717+0000","last_active":"2024-04-26T05:04:46.498769+0000","last_peered":"2024-04-26T05:04:46.498769+0000","last_clean":"2024-04-26T05:04:46.498769+0000","last_became_active":"2024-04-26T05:03:51.793441+0000","last_became_peered":"2024-04-26T05:03:51.793441+0000","last_unstale":"2024-04-26T05:04:46.498769+0000","last_undegraded":"2024-04-26T05:04:46.498769+0000","last_fullsized":"2024-04-26T05:04:46.498769+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:31:44.224541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":42,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.504722+0000","last_change":"2024-04-26T04:55:07.500866+0000","last_active":"2024-04-26T05:04:46.504722+0000","last_peered":"2024-04-26T05:04:46.504722+0000","last_clean":"2024-04-26T05:04:46.504722+0000","last_became_active":"2024-04-26T04:55:07.500516+0000","last_became_peered":"2024-04-26T04:55:07.500516+0000","last_unstale":"2024-04-26T05:04:46.504722+0000","last_undegraded":"2024-04-26T05:04:46.504722+0000","last_fullsized":"2024-04-26T05:04:46.504722+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:10:30.230397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.522349+0000","last_change":"2024-04-26T05:04:47.522349+0000","last_active":"2024-04-26T05:04:47.522349+0000","last_peered":"2024-04-26T05:04:47.522349+0000","last_clean":"2024-04-26T05:04:47.522349+0000","last_became_active":"2024-04-26T05:04:47.521962+0000","last_became_peered":"2024-04-26T05:04:47.521962+0000","last_unstale":"2024-04-26T05:04:47.522349+0000","last_undegraded":"2024-04-26T05:04:47.522349+0000","last_fullsized":"2024-04-26T05:04:47.522349+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:39:10.385626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":440,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.498632+0000","last_change":"2024-04-26T05:02:40.769962+0000","last_active":"2024-04-26T05:04:46.498632+0000","last_peered":"2024-04-26T05:04:46.498632+0000","last_clean":"2024-04-26T05:04:46.498632+0000","last_became_active":"2024-04-26T05:02:40.769150+0000","last_became_peered":"2024-04-26T05:02:40.769150+0000","last_unstale":"2024-04-26T05:04:46.498632+0000","last_undegraded":"2024-04-26T05:04:46.498632+0000","last_fullsized":"2024-04-26T05:04:46.498632+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:51:27.309540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:51:27.309540+0000","last_clean_scrub_stamp":"2024-04-26T04:51:27.309540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:35:02.595664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":64,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.500477+0000","last_change":"2024-04-26T05:04:12.126815+0000","last_active":"2024-04-26T05:04:46.500477+0000","last_peered":"2024-04-26T05:04:46.500477+0000","last_clean":"2024-04-26T05:04:46.500477+0000","last_became_active":"2024-04-26T05:04:12.125959+0000","last_became_peered":"2024-04-26T05:04:12.125959+0000","last_unstale":"2024-04-26T05:04:46.500477+0000","last_undegraded":"2024-04-26T05:04:46.500477+0000","last_fullsized":"2024-04-26T05:04:46.500477+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:00:15.738485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":76,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.506386+0000","last_change":"2024-04-26T05:03:51.803993+0000","last_active":"2024-04-26T05:04:47.506386+0000","last_peered":"2024-04-26T05:04:47.506386+0000","last_clean":"2024-04-26T05:04:47.506386+0000","last_became_active":"2024-04-26T05:03:51.800058+0000","last_became_peered":"2024-04-26T05:03:51.800058+0000","last_unstale":"2024-04-26T05:04:47.506386+0000","last_undegraded":"2024-04-26T05:04:47.506386+0000","last_fullsized":"2024-04-26T05:04:47.506386+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:15:59.299215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":51,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.506978+0000","last_change":"2024-04-26T05:04:12.123623+0000","last_active":"2024-04-26T05:04:47.506978+0000","last_peered":"2024-04-26T05:04:47.506978+0000","last_clean":"2024-04-26T05:04:47.506978+0000","last_became_active":"2024-04-26T05:04:12.123284+0000","last_became_peered":"2024-04-26T05:04:12.123284+0000","last_unstale":"2024-04-26T05:04:47.506978+0000","last_undegraded":"2024-04-26T05:04:47.506978+0000","last_fullsized":"2024-04-26T05:04:47.506978+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:12:47.456887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":81,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.508791+0000","last_change":"2024-04-26T05:04:47.508791+0000","last_active":"2024-04-26T05:04:47.508791+0000","last_peered":"2024-04-26T05:04:47.508791+0000","last_clean":"2024-04-26T05:04:47.508791+0000","last_became_active":"2024-04-26T05:04:47.508464+0000","last_became_peered":"2024-04-26T05:04:47.508464+0000","last_unstale":"2024-04-26T05:04:47.508791+0000","last_undegraded":"2024-04-26T05:04:47.508791+0000","last_fullsized":"2024-04-26T05:04:47.508791+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:40:18.046251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":61,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.515543+0000","last_change":"2024-04-26T05:04:47.515543+0000","last_active":"2024-04-26T05:04:47.515543+0000","last_peered":"2024-04-26T05:04:47.515543+0000","last_clean":"2024-04-26T05:04:47.515543+0000","last_became_active":"2024-04-26T05:04:47.515146+0000","last_became_peered":"2024-04-26T05:04:47.515146+0000","last_unstale":"2024-04-26T05:04:47.515543+0000","last_undegraded":"2024-04-26T05:04:47.515543+0000","last_fullsized":"2024-04-26T05:04:47.515543+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:38:13.919522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":60,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.502562+0000","last_change":"2024-04-26T05:02:40.768748+0000","last_active":"2024-04-26T05:04:46.502562+0000","last_peered":"2024-04-26T05:04:46.502562+0000","last_clean":"2024-04-26T05:04:46.502562+0000","last_became_active":"2024-04-26T05:02:40.768413+0000","last_became_peered":"2024-04-26T05:02:40.768413+0000","last_unstale":"2024-04-26T05:04:46.502562+0000","last_undegraded":"2024-04-26T05:04:46.502562+0000","last_fullsized":"2024-04-26T05:04:46.502562+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:11:33.877754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":64,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.500596+0000","last_change":"2024-04-26T05:03:51.805183+0000","last_active":"2024-04-26T05:04:46.500596+0000","last_peered":"2024-04-26T05:04:46.500596+0000","last_clean":"2024-04-26T05:04:46.500596+0000","last_became_active":"2024-04-26T05:03:51.804789+0000","last_became_peered":"2024-04-26T05:03:51.804789+0000","last_unstale":"2024-04-26T05:04:46.500596+0000","last_undegraded":"2024-04-26T05:04:46.500596+0000","last_fullsized":"2024-04-26T05:04:46.500596+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:05:53.590354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":43,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.506077+0000","last_change":"2024-04-26T05:03:51.805254+0000","last_active":"2024-04-26T05:04:47.506077+0000","last_peered":"2024-04-26T05:04:47.506077+0000","last_clean":"2024-04-26T05:04:47.506077+0000","last_became_active":"2024-04-26T05:03:51.804923+0000","last_became_peered":"2024-04-26T05:03:51.804923+0000","last_unstale":"2024-04-26T05:04:47.506077+0000","last_undegraded":"2024-04-26T05:04:47.506077+0000","last_fullsized":"2024-04-26T05:04:47.506077+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:46:24.941436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":77,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.517100+0000","last_change":"2024-04-26T05:04:47.517100+0000","last_active":"2024-04-26T05:04:47.517100+0000","last_peered":"2024-04-26T05:04:47.517100+0000","last_clean":"2024-04-26T05:04:47.517100+0000","last_became_active":"2024-04-26T05:04:47.516646+0000","last_became_peered":"2024-04-26T05:04:47.516646+0000","last_unstale":"2024-04-26T05:04:47.517100+0000","last_undegraded":"2024-04-26T05:04:47.517100+0000","last_fullsized":"2024-04-26T05:04:47.517100+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:04:59.815879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":81,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.508261+0000","last_change":"2024-04-26T05:03:51.801036+0000","last_active":"2024-04-26T05:04:47.508261+0000","last_peered":"2024-04-26T05:04:47.508261+0000","last_clean":"2024-04-26T05:04:47.508261+0000","last_became_active":"2024-04-26T05:03:51.800724+0000","last_became_peered":"2024-04-26T05:03:51.800724+0000","last_unstale":"2024-04-26T05:04:47.508261+0000","last_undegraded":"2024-04-26T05:04:47.508261+0000","last_fullsized":"2024-04-26T05:04:47.508261+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:09:11.757837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":52,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.516618+0000","last_change":"2024-04-26T05:04:47.516618+0000","last_active":"2024-04-26T05:04:47.516618+0000","last_peered":"2024-04-26T05:04:47.516618+0000","last_clean":"2024-04-26T05:04:47.516618+0000","last_became_active":"2024-04-26T05:04:47.516314+0000","last_became_peered":"2024-04-26T05:04:47.516314+0000","last_unstale":"2024-04-26T05:04:47.516618+0000","last_undegraded":"2024-04-26T05:04:47.516618+0000","last_fullsized":"2024-04-26T05:04:47.516618+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:27:25.679342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":63,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.517784+0000","last_change":"2024-04-26T05:04:47.517784+0000","last_active":"2024-04-26T05:04:47.517784+0000","last_peered":"2024-04-26T05:04:47.517784+0000","last_clean":"2024-04-26T05:04:47.517784+0000","last_became_active":"2024-04-26T05:04:47.517434+0000","last_became_peered":"2024-04-26T05:04:47.517434+0000","last_unstale":"2024-04-26T05:04:47.517784+0000","last_undegraded":"2024-04-26T05:04:47.517784+0000","last_fullsized":"2024-04-26T05:04:47.517784+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:08:54.244848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.504666+0000","last_change":"2024-04-26T05:03:51.806769+0000","last_active":"2024-04-26T05:04:46.504666+0000","last_peered":"2024-04-26T05:04:46.504666+0000","last_clean":"2024-04-26T05:04:46.504666+0000","last_became_active":"2024-04-26T05:03:51.806170+0000","last_became_peered":"2024-04-26T05:03:51.806170+0000","last_unstale":"2024-04-26T05:04:46.504666+0000","last_undegraded":"2024-04-26T05:04:46.504666+0000","last_fullsized":"2024-04-26T05:04:46.504666+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:21:39.768297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":60,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.501127+0000","last_change":"2024-04-26T05:04:12.125834+0000","last_active":"2024-04-26T05:04:46.501127+0000","last_peered":"2024-04-26T05:04:46.501127+0000","last_clean":"2024-04-26T05:04:46.501127+0000","last_became_active":"2024-04-26T05:04:12.125517+0000","last_became_peered":"2024-04-26T05:04:12.125517+0000","last_unstale":"2024-04-26T05:04:46.501127+0000","last_undegraded":"2024-04-26T05:04:46.501127+0000","last_fullsized":"2024-04-26T05:04:46.501127+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:13:16.391364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":76,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.505297+0000","last_change":"2024-04-26T05:04:12.125007+0000","last_active":"2024-04-26T05:04:46.505297+0000","last_peered":"2024-04-26T05:04:46.505297+0000","last_clean":"2024-04-26T05:04:46.505297+0000","last_became_active":"2024-04-26T05:04:12.123926+0000","last_became_peered":"2024-04-26T05:04:12.123926+0000","last_unstale":"2024-04-26T05:04:46.505297+0000","last_undegraded":"2024-04-26T05:04:46.505297+0000","last_fullsized":"2024-04-26T05:04:46.505297+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:24:39.761901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":60,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.504406+0000","last_change":"2024-04-26T05:03:51.805074+0000","last_active":"2024-04-26T05:04:46.504406+0000","last_peered":"2024-04-26T05:04:46.504406+0000","last_clean":"2024-04-26T05:04:46.504406+0000","last_became_active":"2024-04-26T05:03:51.804710+0000","last_became_peered":"2024-04-26T05:03:51.804710+0000","last_unstale":"2024-04-26T05:04:46.504406+0000","last_undegraded":"2024-04-26T05:04:46.504406+0000","last_fullsized":"2024-04-26T05:04:46.504406+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:48:49.853794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":59,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.523083+0000","last_change":"2024-04-26T05:04:47.523083+0000","last_active":"2024-04-26T05:04:47.523083+0000","last_peered":"2024-04-26T05:04:47.523083+0000","last_clean":"2024-04-26T05:04:47.523083+0000","last_became_active":"2024-04-26T05:04:47.522652+0000","last_became_peered":"2024-04-26T05:04:47.522652+0000","last_unstale":"2024-04-26T05:04:47.523083+0000","last_undegraded":"2024-04-26T05:04:47.523083+0000","last_fullsized":"2024-04-26T05:04:47.523083+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:05:30.198750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":60,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.514735+0000","last_change":"2024-04-26T05:04:47.514735+0000","last_active":"2024-04-26T05:04:47.514735+0000","last_peered":"2024-04-26T05:04:47.514735+0000","last_clean":"2024-04-26T05:04:47.514735+0000","last_became_active":"2024-04-26T05:04:47.514362+0000","last_became_peered":"2024-04-26T05:04:47.514362+0000","last_unstale":"2024-04-26T05:04:47.514735+0000","last_undegraded":"2024-04-26T05:04:47.514735+0000","last_fullsized":"2024-04-26T05:04:47.514735+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:41:30.261068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":63,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.501825+0000","last_change":"2024-04-26T05:03:51.801392+0000","last_active":"2024-04-26T05:04:47.501825+0000","last_peered":"2024-04-26T05:04:47.501825+0000","last_clean":"2024-04-26T05:04:47.501825+0000","last_became_active":"2024-04-26T05:03:51.800856+0000","last_became_peered":"2024-04-26T05:03:51.800856+0000","last_unstale":"2024-04-26T05:04:47.501825+0000","last_undegraded":"2024-04-26T05:04:47.501825+0000","last_fullsized":"2024-04-26T05:04:47.501825+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:05:58.425337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":60,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.521951+0000","last_change":"2024-04-26T05:04:47.521951+0000","last_active":"2024-04-26T05:04:47.521951+0000","last_peered":"2024-04-26T05:04:47.521951+0000","last_clean":"2024-04-26T05:04:47.521951+0000","last_became_active":"2024-04-26T05:04:47.521588+0000","last_became_peered":"2024-04-26T05:04:47.521588+0000","last_unstale":"2024-04-26T05:04:47.521951+0000","last_undegraded":"2024-04-26T05:04:47.521951+0000","last_fullsized":"2024-04-26T05:04:47.521951+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:04:26.827180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.504251+0000","last_change":"2024-04-26T05:03:51.803035+0000","last_active":"2024-04-26T05:04:46.504251+0000","last_peered":"2024-04-26T05:04:46.504251+0000","last_clean":"2024-04-26T05:04:46.504251+0000","last_became_active":"2024-04-26T05:03:51.802703+0000","last_became_peered":"2024-04-26T05:03:51.802703+0000","last_unstale":"2024-04-26T05:04:46.504251+0000","last_undegraded":"2024-04-26T05:04:46.504251+0000","last_fullsized":"2024-04-26T05:04:46.504251+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:01:22.472530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":43,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.498208+0000","last_change":"2024-04-26T04:55:07.451952+0000","last_active":"2024-04-26T05:04:46.498208+0000","last_peered":"2024-04-26T05:04:46.498208+0000","last_clean":"2024-04-26T05:04:46.498208+0000","last_became_active":"2024-04-26T04:55:07.451660+0000","last_became_peered":"2024-04-26T04:55:07.451660+0000","last_unstale":"2024-04-26T05:04:46.498208+0000","last_undegraded":"2024-04-26T05:04:46.498208+0000","last_fullsized":"2024-04-26T05:04:46.498208+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:36:14.683401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":76,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.518564+0000","last_change":"2024-04-26T05:04:47.518564+0000","last_active":"2024-04-26T05:04:47.518564+0000","last_peered":"2024-04-26T05:04:47.518564+0000","last_clean":"2024-04-26T05:04:47.518564+0000","last_became_active":"2024-04-26T05:04:47.518099+0000","last_became_peered":"2024-04-26T05:04:47.518099+0000","last_unstale":"2024-04-26T05:04:47.518564+0000","last_undegraded":"2024-04-26T05:04:47.518564+0000","last_fullsized":"2024-04-26T05:04:47.518564+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:00:23.156348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":61,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.498095+0000","last_change":"2024-04-26T05:03:51.806785+0000","last_active":"2024-04-26T05:04:46.498095+0000","last_peered":"2024-04-26T05:04:46.498095+0000","last_clean":"2024-04-26T05:04:46.498095+0000","last_became_active":"2024-04-26T05:03:51.806370+0000","last_became_peered":"2024-04-26T05:03:51.806370+0000","last_unstale":"2024-04-26T05:04:46.498095+0000","last_undegraded":"2024-04-26T05:04:46.498095+0000","last_fullsized":"2024-04-26T05:04:46.498095+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:51:41.841527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":42,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.500014+0000","last_change":"2024-04-26T04:55:07.500711+0000","last_active":"2024-04-26T05:04:46.500014+0000","last_peered":"2024-04-26T05:04:46.500014+0000","last_clean":"2024-04-26T05:04:46.500014+0000","last_became_active":"2024-04-26T04:55:07.500428+0000","last_became_peered":"2024-04-26T04:55:07.500428+0000","last_unstale":"2024-04-26T05:04:46.500014+0000","last_undegraded":"2024-04-26T05:04:46.500014+0000","last_fullsized":"2024-04-26T05:04:46.500014+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:07:04.979529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":60,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.498800+0000","last_change":"2024-04-26T05:02:40.768192+0000","last_active":"2024-04-26T05:04:46.498800+0000","last_peered":"2024-04-26T05:04:46.498800+0000","last_clean":"2024-04-26T05:04:46.498800+0000","last_became_active":"2024-04-26T05:02:40.767729+0000","last_became_peered":"2024-04-26T05:02:40.767729+0000","last_unstale":"2024-04-26T05:04:46.498800+0000","last_undegraded":"2024-04-26T05:04:46.498800+0000","last_fullsized":"2024-04-26T05:04:46.498800+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:42:52.054207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":76,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.519627+0000","last_change":"2024-04-26T05:04:47.519627+0000","last_active":"2024-04-26T05:04:47.519627+0000","last_peered":"2024-04-26T05:04:47.519627+0000","last_clean":"2024-04-26T05:04:47.519627+0000","last_became_active":"2024-04-26T05:04:47.518930+0000","last_became_peered":"2024-04-26T05:04:47.518930+0000","last_unstale":"2024-04-26T05:04:47.519627+0000","last_undegraded":"2024-04-26T05:04:47.519627+0000","last_fullsized":"2024-04-26T05:04:47.519627+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:16:01.807071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":68,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.502979+0000","last_change":"2024-04-26T05:02:40.768573+0000","last_active":"2024-04-26T05:04:46.502979+0000","last_peered":"2024-04-26T05:04:46.502979+0000","last_clean":"2024-04-26T05:04:46.502979+0000","last_became_active":"2024-04-26T05:02:40.768038+0000","last_became_peered":"2024-04-26T05:02:40.768038+0000","last_unstale":"2024-04-26T05:04:46.502979+0000","last_undegraded":"2024-04-26T05:04:46.502979+0000","last_fullsized":"2024-04-26T05:04:46.502979+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:17:09.118193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":63,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.518264+0000","last_change":"2024-04-26T05:04:47.517331+0000","last_active":"2024-04-26T05:04:47.518264+0000","last_peered":"2024-04-26T05:04:47.518264+0000","last_clean":"2024-04-26T05:04:47.518264+0000","last_became_active":"2024-04-26T05:04:47.517012+0000","last_became_peered":"2024-04-26T05:04:47.517012+0000","last_unstale":"2024-04-26T05:04:47.518264+0000","last_undegraded":"2024-04-26T05:04:47.518264+0000","last_fullsized":"2024-04-26T05:04:47.518264+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:17:54.745143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":42,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.497124+0000","last_change":"2024-04-26T04:55:07.452729+0000","last_active":"2024-04-26T05:04:46.497124+0000","last_peered":"2024-04-26T05:04:46.497124+0000","last_clean":"2024-04-26T05:04:46.497124+0000","last_became_active":"2024-04-26T04:55:07.452426+0000","last_became_peered":"2024-04-26T04:55:07.452426+0000","last_unstale":"2024-04-26T05:04:46.497124+0000","last_undegraded":"2024-04-26T05:04:46.497124+0000","last_fullsized":"2024-04-26T05:04:46.497124+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:59:48.509688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":59,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.517023+0000","last_change":"2024-04-26T05:04:47.517023+0000","last_active":"2024-04-26T05:04:47.517023+0000","last_peered":"2024-04-26T05:04:47.517023+0000","last_clean":"2024-04-26T05:04:47.517023+0000","last_became_active":"2024-04-26T05:04:47.516646+0000","last_became_peered":"2024-04-26T05:04:47.516646+0000","last_unstale":"2024-04-26T05:04:47.517023+0000","last_undegraded":"2024-04-26T05:04:47.517023+0000","last_fullsized":"2024-04-26T05:04:47.517023+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:09:28.891107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":62,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.510533+0000","last_change":"2024-04-26T05:04:47.510190+0000","last_active":"2024-04-26T05:04:47.510533+0000","last_peered":"2024-04-26T05:04:47.510533+0000","last_clean":"2024-04-26T05:04:47.510533+0000","last_became_active":"2024-04-26T05:04:47.509754+0000","last_became_peered":"2024-04-26T05:04:47.509754+0000","last_unstale":"2024-04-26T05:04:47.510533+0000","last_undegraded":"2024-04-26T05:04:47.510533+0000","last_fullsized":"2024-04-26T05:04:47.510533+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:21:14.228592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":62,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.518993+0000","last_change":"2024-04-26T05:04:47.518993+0000","last_active":"2024-04-26T05:04:47.518993+0000","last_peered":"2024-04-26T05:04:47.518993+0000","last_clean":"2024-04-26T05:04:47.518993+0000","last_became_active":"2024-04-26T05:04:47.518384+0000","last_became_peered":"2024-04-26T05:04:47.518384+0000","last_unstale":"2024-04-26T05:04:47.518993+0000","last_undegraded":"2024-04-26T05:04:47.518993+0000","last_fullsized":"2024-04-26T05:04:47.518993+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:27:00.437340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":61,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.497511+0000","last_change":"2024-04-26T05:04:12.124365+0000","last_active":"2024-04-26T05:04:46.497511+0000","last_peered":"2024-04-26T05:04:46.497511+0000","last_clean":"2024-04-26T05:04:46.497511+0000","last_became_active":"2024-04-26T05:04:12.123898+0000","last_became_peered":"2024-04-26T05:04:12.123898+0000","last_unstale":"2024-04-26T05:04:46.497511+0000","last_undegraded":"2024-04-26T05:04:46.497511+0000","last_fullsized":"2024-04-26T05:04:46.497511+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:04:45.820110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.503106+0000","last_change":"2024-04-26T05:03:51.802338+0000","last_active":"2024-04-26T05:04:46.503106+0000","last_peered":"2024-04-26T05:04:46.503106+0000","last_clean":"2024-04-26T05:04:46.503106+0000","last_became_active":"2024-04-26T05:03:51.802024+0000","last_became_peered":"2024-04-26T05:03:51.802024+0000","last_unstale":"2024-04-26T05:04:46.503106+0000","last_undegraded":"2024-04-26T05:04:46.503106+0000","last_fullsized":"2024-04-26T05:04:46.503106+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:08:37.594439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":65,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.505050+0000","last_change":"2024-04-26T04:55:07.451574+0000","last_active":"2024-04-26T05:04:46.505050+0000","last_peered":"2024-04-26T05:04:46.505050+0000","last_clean":"2024-04-26T05:04:46.505050+0000","last_became_active":"2024-04-26T04:55:07.451154+0000","last_became_peered":"2024-04-26T04:55:07.451154+0000","last_unstale":"2024-04-26T05:04:46.505050+0000","last_undegraded":"2024-04-26T05:04:46.505050+0000","last_fullsized":"2024-04-26T05:04:46.505050+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:56:04.497071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":56,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.500605+0000","last_change":"2024-04-26T05:04:12.126008+0000","last_active":"2024-04-26T05:04:46.500605+0000","last_peered":"2024-04-26T05:04:46.500605+0000","last_clean":"2024-04-26T05:04:46.500605+0000","last_became_active":"2024-04-26T05:04:12.125566+0000","last_became_peered":"2024-04-26T05:04:12.125566+0000","last_unstale":"2024-04-26T05:04:46.500605+0000","last_undegraded":"2024-04-26T05:04:46.500605+0000","last_fullsized":"2024-04-26T05:04:46.500605+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:32:57.921469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":62,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.518237+0000","last_change":"2024-04-26T05:04:47.518237+0000","last_active":"2024-04-26T05:04:47.518237+0000","last_peered":"2024-04-26T05:04:47.518237+0000","last_clean":"2024-04-26T05:04:47.518237+0000","last_became_active":"2024-04-26T05:04:47.517876+0000","last_became_peered":"2024-04-26T05:04:47.517876+0000","last_unstale":"2024-04-26T05:04:47.518237+0000","last_undegraded":"2024-04-26T05:04:47.518237+0000","last_fullsized":"2024-04-26T05:04:47.518237+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:21:16.591946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.523513+0000","last_change":"2024-04-26T05:04:47.523513+0000","last_active":"2024-04-26T05:04:47.523513+0000","last_peered":"2024-04-26T05:04:47.523513+0000","last_clean":"2024-04-26T05:04:47.523513+0000","last_became_active":"2024-04-26T05:04:47.523248+0000","last_became_peered":"2024-04-26T05:04:47.523248+0000","last_unstale":"2024-04-26T05:04:47.523513+0000","last_undegraded":"2024-04-26T05:04:47.523513+0000","last_fullsized":"2024-04-26T05:04:47.523513+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:28:42.069788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":60,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.500130+0000","last_change":"2024-04-26T05:03:51.805729+0000","last_active":"2024-04-26T05:04:46.500130+0000","last_peered":"2024-04-26T05:04:46.500130+0000","last_clean":"2024-04-26T05:04:46.500130+0000","last_became_active":"2024-04-26T05:03:51.805401+0000","last_became_peered":"2024-04-26T05:03:51.805401+0000","last_unstale":"2024-04-26T05:04:46.500130+0000","last_undegraded":"2024-04-26T05:04:46.500130+0000","last_fullsized":"2024-04-26T05:04:46.500130+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:05:36.172467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":62,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.521896+0000","last_change":"2024-04-26T05:04:47.521896+0000","last_active":"2024-04-26T05:04:47.521896+0000","last_peered":"2024-04-26T05:04:47.521896+0000","last_clean":"2024-04-26T05:04:47.521896+0000","last_became_active":"2024-04-26T05:04:47.521441+0000","last_became_peered":"2024-04-26T05:04:47.521441+0000","last_unstale":"2024-04-26T05:04:47.521896+0000","last_undegraded":"2024-04-26T05:04:47.521896+0000","last_fullsized":"2024-04-26T05:04:47.521896+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:47:15.628754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":62,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.517199+0000","last_change":"2024-04-26T05:04:47.516799+0000","last_active":"2024-04-26T05:04:47.517199+0000","last_peered":"2024-04-26T05:04:47.517199+0000","last_clean":"2024-04-26T05:04:47.517199+0000","last_became_active":"2024-04-26T05:04:47.516430+0000","last_became_peered":"2024-04-26T05:04:47.516430+0000","last_unstale":"2024-04-26T05:04:47.517199+0000","last_undegraded":"2024-04-26T05:04:47.517199+0000","last_fullsized":"2024-04-26T05:04:47.517199+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:34:03.727441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.505720+0000","last_change":"2024-04-26T05:03:51.800451+0000","last_active":"2024-04-26T05:04:46.505720+0000","last_peered":"2024-04-26T05:04:46.505720+0000","last_clean":"2024-04-26T05:04:46.505720+0000","last_became_active":"2024-04-26T05:03:51.800110+0000","last_became_peered":"2024-04-26T05:03:51.800110+0000","last_unstale":"2024-04-26T05:04:46.505720+0000","last_undegraded":"2024-04-26T05:04:46.505720+0000","last_fullsized":"2024-04-26T05:04:46.505720+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:03:47.250736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":80,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.513933+0000","last_change":"2024-04-26T05:04:47.513933+0000","last_active":"2024-04-26T05:04:47.513933+0000","last_peered":"2024-04-26T05:04:47.513933+0000","last_clean":"2024-04-26T05:04:47.513933+0000","last_became_active":"2024-04-26T05:04:47.512605+0000","last_became_peered":"2024-04-26T05:04:47.512605+0000","last_unstale":"2024-04-26T05:04:47.513933+0000","last_undegraded":"2024-04-26T05:04:47.513933+0000","last_fullsized":"2024-04-26T05:04:47.513933+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:56:26.131873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":63,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.517826+0000","last_change":"2024-04-26T05:04:47.517826+0000","last_active":"2024-04-26T05:04:47.517826+0000","last_peered":"2024-04-26T05:04:47.517826+0000","last_clean":"2024-04-26T05:04:47.517826+0000","last_became_active":"2024-04-26T05:04:47.517502+0000","last_became_peered":"2024-04-26T05:04:47.517502+0000","last_unstale":"2024-04-26T05:04:47.517826+0000","last_undegraded":"2024-04-26T05:04:47.517826+0000","last_fullsized":"2024-04-26T05:04:47.517826+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:18:34.533782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":51,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.513174+0000","last_change":"2024-04-26T05:04:47.513174+0000","last_active":"2024-04-26T05:04:47.513174+0000","last_peered":"2024-04-26T05:04:47.513174+0000","last_clean":"2024-04-26T05:04:47.513174+0000","last_became_active":"2024-04-26T05:04:47.512833+0000","last_became_peered":"2024-04-26T05:04:47.512833+0000","last_unstale":"2024-04-26T05:04:47.513174+0000","last_undegraded":"2024-04-26T05:04:47.513174+0000","last_fullsized":"2024-04-26T05:04:47.513174+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:16:42.884641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":62,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.514646+0000","last_change":"2024-04-26T05:04:47.514646+0000","last_active":"2024-04-26T05:04:47.514646+0000","last_peered":"2024-04-26T05:04:47.514646+0000","last_clean":"2024-04-26T05:04:47.514646+0000","last_became_active":"2024-04-26T05:04:47.514291+0000","last_became_peered":"2024-04-26T05:04:47.514291+0000","last_unstale":"2024-04-26T05:04:47.514646+0000","last_undegraded":"2024-04-26T05:04:47.514646+0000","last_fullsized":"2024-04-26T05:04:47.514646+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:01:50.135758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":78,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.501039+0000","last_change":"2024-04-26T05:04:12.124882+0000","last_active":"2024-04-26T05:04:46.501039+0000","last_peered":"2024-04-26T05:04:46.501039+0000","last_clean":"2024-04-26T05:04:46.501039+0000","last_became_active":"2024-04-26T05:04:12.124468+0000","last_became_peered":"2024-04-26T05:04:12.124468+0000","last_unstale":"2024-04-26T05:04:46.501039+0000","last_undegraded":"2024-04-26T05:04:46.501039+0000","last_fullsized":"2024-04-26T05:04:46.501039+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:30:14.534509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1053,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201431,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311308,"kb_used_data":1664,"kb_used_omap":1,"kb_used_meta":309630,"kb_avail":93433844,"statfs":{"total":95995035648,"available":95676256256,"internally_reserved":0,"allocated":1703936,"data_stored":1151251,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317061464},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463068,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299012,"kb_used_data":1648,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446140,"statfs":{"total":95995035648,"available":95688847360,"internally_reserved":0,"allocated":1687552,"data_stored":1150890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":89,"seq":382252089368,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28592,"kb_used_data":1060,"kb_used_omap":8,"kb_used_meta":27511,"kb_avail":93716560,"statfs":{"total":95995035648,"available":95965757440,"internally_reserved":0,"allocated":1085440,"data_stored":556274,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8667,"internal_metadata":28171813},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":84,"seq":360777252902,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":29176,"kb_used_data":1644,"kb_used_omap":18,"kb_used_meta":27501,"kb_avail":93715976,"statfs":{"total":95995035648,"available":95965159424,"internally_reserved":0,"allocated":1683456,"data_stored":1146969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19374,"internal_metadata":28161106},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":94,"seq":403726925844,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28604,"kb_used_data":1068,"kb_used_omap":47,"kb_used_meta":27472,"kb_avail":93716548,"statfs":{"total":95995035648,"available":95965745152,"internally_reserved":0,"allocated":1093632,"data_stored":556878,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49111,"internal_metadata":28131369},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":99,"seq":425201762317,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28596,"kb_used_data":1064,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93716556,"statfs":{"total":95995035648,"available":95965753344,"internally_reserved":0,"allocated":1089536,"data_stored":556356,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7696,"internal_metadata":28172784},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986341,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302536,"kb_used_data":1084,"kb_used_omap":1,"kb_used_meta":301438,"kb_avail":93442616,"statfs":{"total":95995035648,"available":95685238784,"internally_reserved":0,"allocated":1110016,"data_stored":561136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308672856},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724705,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306620,"kb_used_data":1064,"kb_used_omap":1,"kb_used_meta":305534,"kb_avail":93438532,"statfs":{"total":95995035648,"available":95681056768,"internally_reserved":0,"allocated":1089536,"data_stored":556563,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312867160},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1053,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-26T05:05:44.915 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-26T05:05:44.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:44 smithi116 ceph-mon[85502]: pgmap v231: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:45.183 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:45.972 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:45 smithi012 ceph-mon[91852]: from='client.34578 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T05:05:45.972 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:45 smithi012 ceph-mon[95978]: from='client.34578 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T05:05:45.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:45 smithi116 ceph-mon[85502]: from='client.34578 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T05:05:46.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:46 smithi116 ceph-mon[85502]: pgmap v232: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:47.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:46 smithi012 ceph-mon[91852]: pgmap v232: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:47.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:46 smithi012 ceph-mon[95978]: pgmap v232: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:47.065 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:05:47.066 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2024-04-26T05:05:47.838 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":232,"stamp":"2024-04-26T05:05:45.471344+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595898,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1334444,"kb_used_data":10296,"kb_used_omap":89,"kb_used_meta":1323942,"kb_avail":748626772,"statfs":{"total":767960285184,"available":766593814528,"internally_reserved":0,"allocated":10543104,"data_stored":6236317,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":91664,"internal_metadata":1355717104},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002801"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.501003+0000","last_change":"2024-04-26T05:03:51.802774+0000","last_active":"2024-04-26T05:04:46.501003+0000","last_peered":"2024-04-26T05:04:46.501003+0000","last_clean":"2024-04-26T05:04:46.501003+0000","last_became_active":"2024-04-26T05:03:51.802166+0000","last_became_peered":"2024-04-26T05:03:51.802166+0000","last_unstale":"2024-04-26T05:04:46.501003+0000","last_undegraded":"2024-04-26T05:04:46.501003+0000","last_fullsized":"2024-04-26T05:04:46.501003+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:19:02.626403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":66,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.502100+0000","last_change":"2024-04-26T05:04:12.124459+0000","last_active":"2024-04-26T05:04:47.502100+0000","last_peered":"2024-04-26T05:04:47.502100+0000","last_clean":"2024-04-26T05:04:47.502100+0000","last_became_active":"2024-04-26T05:04:12.123993+0000","last_became_peered":"2024-04-26T05:04:12.123993+0000","last_unstale":"2024-04-26T05:04:47.502100+0000","last_undegraded":"2024-04-26T05:04:47.502100+0000","last_fullsized":"2024-04-26T05:04:47.502100+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:50:29.431399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":83,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.513958+0000","last_change":"2024-04-26T05:04:47.513958+0000","last_active":"2024-04-26T05:04:47.513958+0000","last_peered":"2024-04-26T05:04:47.513958+0000","last_clean":"2024-04-26T05:04:47.513958+0000","last_became_active":"2024-04-26T05:04:47.513425+0000","last_became_peered":"2024-04-26T05:04:47.513425+0000","last_unstale":"2024-04-26T05:04:47.513958+0000","last_undegraded":"2024-04-26T05:04:47.513958+0000","last_fullsized":"2024-04-26T05:04:47.513958+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:12:41.169186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":68,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.522311+0000","last_change":"2024-04-26T05:04:47.522311+0000","last_active":"2024-04-26T05:04:47.522311+0000","last_peered":"2024-04-26T05:04:47.522311+0000","last_clean":"2024-04-26T05:04:47.522311+0000","last_became_active":"2024-04-26T05:04:47.521599+0000","last_became_peered":"2024-04-26T05:04:47.521599+0000","last_unstale":"2024-04-26T05:04:47.522311+0000","last_undegraded":"2024-04-26T05:04:47.522311+0000","last_fullsized":"2024-04-26T05:04:47.522311+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:32:57.457648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":76,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.509750+0000","last_change":"2024-04-26T05:04:47.509750+0000","last_active":"2024-04-26T05:04:47.509750+0000","last_peered":"2024-04-26T05:04:47.509750+0000","last_clean":"2024-04-26T05:04:47.509750+0000","last_became_active":"2024-04-26T05:04:47.509377+0000","last_became_peered":"2024-04-26T05:04:47.509377+0000","last_unstale":"2024-04-26T05:04:47.509750+0000","last_undegraded":"2024-04-26T05:04:47.509750+0000","last_fullsized":"2024-04-26T05:04:47.509750+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:05:14.524534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":45,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.501340+0000","last_change":"2024-04-26T04:55:07.445793+0000","last_active":"2024-04-26T05:04:46.501340+0000","last_peered":"2024-04-26T05:04:46.501340+0000","last_clean":"2024-04-26T05:04:46.501340+0000","last_became_active":"2024-04-26T04:55:07.445496+0000","last_became_peered":"2024-04-26T04:55:07.445496+0000","last_unstale":"2024-04-26T05:04:46.501340+0000","last_undegraded":"2024-04-26T05:04:46.501340+0000","last_fullsized":"2024-04-26T05:04:46.501340+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:53:56.081693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":86,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.509867+0000","last_change":"2024-04-26T05:04:47.509867+0000","last_active":"2024-04-26T05:04:47.509867+0000","last_peered":"2024-04-26T05:04:47.509867+0000","last_clean":"2024-04-26T05:04:47.509867+0000","last_became_active":"2024-04-26T05:04:47.509500+0000","last_became_peered":"2024-04-26T05:04:47.509500+0000","last_unstale":"2024-04-26T05:04:47.509867+0000","last_undegraded":"2024-04-26T05:04:47.509867+0000","last_fullsized":"2024-04-26T05:04:47.509867+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:58:05.754843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":62,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.500044+0000","last_change":"2024-04-26T05:03:51.801296+0000","last_active":"2024-04-26T05:04:46.500044+0000","last_peered":"2024-04-26T05:04:46.500044+0000","last_clean":"2024-04-26T05:04:46.500044+0000","last_became_active":"2024-04-26T05:03:51.800904+0000","last_became_peered":"2024-04-26T05:03:51.800904+0000","last_unstale":"2024-04-26T05:04:46.500044+0000","last_undegraded":"2024-04-26T05:04:46.500044+0000","last_fullsized":"2024-04-26T05:04:46.500044+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:23:49.534793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":80,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.520013+0000","last_change":"2024-04-26T05:04:47.520013+0000","last_active":"2024-04-26T05:04:47.520013+0000","last_peered":"2024-04-26T05:04:47.520013+0000","last_clean":"2024-04-26T05:04:47.520013+0000","last_became_active":"2024-04-26T05:04:47.519664+0000","last_became_peered":"2024-04-26T05:04:47.519664+0000","last_unstale":"2024-04-26T05:04:47.520013+0000","last_undegraded":"2024-04-26T05:04:47.520013+0000","last_fullsized":"2024-04-26T05:04:47.520013+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:11:24.790879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":57,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.501904+0000","last_change":"2024-04-26T05:02:40.771411+0000","last_active":"2024-04-26T05:04:47.501904+0000","last_peered":"2024-04-26T05:04:47.501904+0000","last_clean":"2024-04-26T05:04:47.501904+0000","last_became_active":"2024-04-26T05:02:40.770506+0000","last_became_peered":"2024-04-26T05:02:40.770506+0000","last_unstale":"2024-04-26T05:04:47.501904+0000","last_undegraded":"2024-04-26T05:04:47.501904+0000","last_fullsized":"2024-04-26T05:04:47.501904+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:49:20.399132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":42,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.501215+0000","last_change":"2024-04-26T04:55:07.474089+0000","last_active":"2024-04-26T05:04:46.501215+0000","last_peered":"2024-04-26T05:04:46.501215+0000","last_clean":"2024-04-26T05:04:46.501215+0000","last_became_active":"2024-04-26T04:55:07.473624+0000","last_became_peered":"2024-04-26T04:55:07.473624+0000","last_unstale":"2024-04-26T05:04:46.501215+0000","last_undegraded":"2024-04-26T05:04:46.501215+0000","last_fullsized":"2024-04-26T05:04:46.501215+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:15:47.507462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":66,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.501978+0000","last_change":"2024-04-26T05:03:51.806178+0000","last_active":"2024-04-26T05:04:47.501978+0000","last_peered":"2024-04-26T05:04:47.501978+0000","last_clean":"2024-04-26T05:04:47.501978+0000","last_became_active":"2024-04-26T05:03:51.805803+0000","last_became_peered":"2024-04-26T05:03:51.805803+0000","last_unstale":"2024-04-26T05:04:47.501978+0000","last_undegraded":"2024-04-26T05:04:47.501978+0000","last_fullsized":"2024-04-26T05:04:47.501978+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:54:57.751726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":64,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.505722+0000","last_change":"2024-04-26T05:04:12.123626+0000","last_active":"2024-04-26T05:04:46.505722+0000","last_peered":"2024-04-26T05:04:46.505722+0000","last_clean":"2024-04-26T05:04:46.505722+0000","last_became_active":"2024-04-26T05:04:12.123268+0000","last_became_peered":"2024-04-26T05:04:12.123268+0000","last_unstale":"2024-04-26T05:04:46.505722+0000","last_undegraded":"2024-04-26T05:04:46.505722+0000","last_fullsized":"2024-04-26T05:04:46.505722+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:11:41.914165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":57,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.516569+0000","last_change":"2024-04-26T05:04:47.516569+0000","last_active":"2024-04-26T05:04:47.516569+0000","last_peered":"2024-04-26T05:04:47.516569+0000","last_clean":"2024-04-26T05:04:47.516569+0000","last_became_active":"2024-04-26T05:04:47.516062+0000","last_became_peered":"2024-04-26T05:04:47.516062+0000","last_unstale":"2024-04-26T05:04:47.516569+0000","last_undegraded":"2024-04-26T05:04:47.516569+0000","last_fullsized":"2024-04-26T05:04:47.516569+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:43:44.271116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":42,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.507556+0000","last_change":"2024-04-26T05:03:51.807810+0000","last_active":"2024-04-26T05:04:47.507556+0000","last_peered":"2024-04-26T05:04:47.507556+0000","last_clean":"2024-04-26T05:04:47.507556+0000","last_became_active":"2024-04-26T05:03:51.805285+0000","last_became_peered":"2024-04-26T05:03:51.805285+0000","last_unstale":"2024-04-26T05:04:47.507556+0000","last_undegraded":"2024-04-26T05:04:47.507556+0000","last_fullsized":"2024-04-26T05:04:47.507556+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:39:43.835533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":60,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.519997+0000","last_change":"2024-04-26T05:04:47.519997+0000","last_active":"2024-04-26T05:04:47.519997+0000","last_peered":"2024-04-26T05:04:47.519997+0000","last_clean":"2024-04-26T05:04:47.519997+0000","last_became_active":"2024-04-26T05:04:47.519403+0000","last_became_peered":"2024-04-26T05:04:47.519403+0000","last_unstale":"2024-04-26T05:04:47.519997+0000","last_undegraded":"2024-04-26T05:04:47.519997+0000","last_fullsized":"2024-04-26T05:04:47.519997+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:20:46.301349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":54,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.501680+0000","last_change":"2024-04-26T05:02:40.772049+0000","last_active":"2024-04-26T05:04:47.501680+0000","last_peered":"2024-04-26T05:04:47.501680+0000","last_clean":"2024-04-26T05:04:47.501680+0000","last_became_active":"2024-04-26T05:02:40.771105+0000","last_became_peered":"2024-04-26T05:02:40.771105+0000","last_unstale":"2024-04-26T05:04:47.501680+0000","last_undegraded":"2024-04-26T05:04:47.501680+0000","last_fullsized":"2024-04-26T05:04:47.501680+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:29:57.754882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":51,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.508294+0000","last_change":"2024-04-26T05:04:12.126757+0000","last_active":"2024-04-26T05:04:47.508294+0000","last_peered":"2024-04-26T05:04:47.508294+0000","last_clean":"2024-04-26T05:04:47.508294+0000","last_became_active":"2024-04-26T05:04:12.126449+0000","last_became_peered":"2024-04-26T05:04:12.126449+0000","last_unstale":"2024-04-26T05:04:47.508294+0000","last_undegraded":"2024-04-26T05:04:47.508294+0000","last_fullsized":"2024-04-26T05:04:47.508294+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:09:15.930209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":63,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.498093+0000","last_change":"2024-04-26T05:02:40.767283+0000","last_active":"2024-04-26T05:04:46.498093+0000","last_peered":"2024-04-26T05:04:46.498093+0000","last_clean":"2024-04-26T05:04:46.498093+0000","last_became_active":"2024-04-26T05:02:40.766809+0000","last_became_peered":"2024-04-26T05:02:40.766809+0000","last_unstale":"2024-04-26T05:04:46.498093+0000","last_undegraded":"2024-04-26T05:04:46.498093+0000","last_fullsized":"2024-04-26T05:04:46.498093+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:09:39.752228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":72,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.502081+0000","last_change":"2024-04-26T05:04:12.124689+0000","last_active":"2024-04-26T05:04:47.502081+0000","last_peered":"2024-04-26T05:04:47.502081+0000","last_clean":"2024-04-26T05:04:47.502081+0000","last_became_active":"2024-04-26T05:04:12.124354+0000","last_became_peered":"2024-04-26T05:04:12.124354+0000","last_unstale":"2024-04-26T05:04:47.502081+0000","last_undegraded":"2024-04-26T05:04:47.502081+0000","last_fullsized":"2024-04-26T05:04:47.502081+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:54:42.387778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":62,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.513467+0000","last_change":"2024-04-26T05:04:47.513467+0000","last_active":"2024-04-26T05:04:47.513467+0000","last_peered":"2024-04-26T05:04:47.513467+0000","last_clean":"2024-04-26T05:04:47.513467+0000","last_became_active":"2024-04-26T05:04:47.513152+0000","last_became_peered":"2024-04-26T05:04:47.513152+0000","last_unstale":"2024-04-26T05:04:47.513467+0000","last_undegraded":"2024-04-26T05:04:47.513467+0000","last_fullsized":"2024-04-26T05:04:47.513467+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:12:13.750681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":80,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.498769+0000","last_change":"2024-04-26T05:03:51.793717+0000","last_active":"2024-04-26T05:04:46.498769+0000","last_peered":"2024-04-26T05:04:46.498769+0000","last_clean":"2024-04-26T05:04:46.498769+0000","last_became_active":"2024-04-26T05:03:51.793441+0000","last_became_peered":"2024-04-26T05:03:51.793441+0000","last_unstale":"2024-04-26T05:04:46.498769+0000","last_undegraded":"2024-04-26T05:04:46.498769+0000","last_fullsized":"2024-04-26T05:04:46.498769+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:31:44.224541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":42,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.504722+0000","last_change":"2024-04-26T04:55:07.500866+0000","last_active":"2024-04-26T05:04:46.504722+0000","last_peered":"2024-04-26T05:04:46.504722+0000","last_clean":"2024-04-26T05:04:46.504722+0000","last_became_active":"2024-04-26T04:55:07.500516+0000","last_became_peered":"2024-04-26T04:55:07.500516+0000","last_unstale":"2024-04-26T05:04:46.504722+0000","last_undegraded":"2024-04-26T05:04:46.504722+0000","last_fullsized":"2024-04-26T05:04:46.504722+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:10:30.230397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.522349+0000","last_change":"2024-04-26T05:04:47.522349+0000","last_active":"2024-04-26T05:04:47.522349+0000","last_peered":"2024-04-26T05:04:47.522349+0000","last_clean":"2024-04-26T05:04:47.522349+0000","last_became_active":"2024-04-26T05:04:47.521962+0000","last_became_peered":"2024-04-26T05:04:47.521962+0000","last_unstale":"2024-04-26T05:04:47.522349+0000","last_undegraded":"2024-04-26T05:04:47.522349+0000","last_fullsized":"2024-04-26T05:04:47.522349+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:39:10.385626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":440,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.498632+0000","last_change":"2024-04-26T05:02:40.769962+0000","last_active":"2024-04-26T05:04:46.498632+0000","last_peered":"2024-04-26T05:04:46.498632+0000","last_clean":"2024-04-26T05:04:46.498632+0000","last_became_active":"2024-04-26T05:02:40.769150+0000","last_became_peered":"2024-04-26T05:02:40.769150+0000","last_unstale":"2024-04-26T05:04:46.498632+0000","last_undegraded":"2024-04-26T05:04:46.498632+0000","last_fullsized":"2024-04-26T05:04:46.498632+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:51:27.309540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:51:27.309540+0000","last_clean_scrub_stamp":"2024-04-26T04:51:27.309540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:35:02.595664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":64,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.500477+0000","last_change":"2024-04-26T05:04:12.126815+0000","last_active":"2024-04-26T05:04:46.500477+0000","last_peered":"2024-04-26T05:04:46.500477+0000","last_clean":"2024-04-26T05:04:46.500477+0000","last_became_active":"2024-04-26T05:04:12.125959+0000","last_became_peered":"2024-04-26T05:04:12.125959+0000","last_unstale":"2024-04-26T05:04:46.500477+0000","last_undegraded":"2024-04-26T05:04:46.500477+0000","last_fullsized":"2024-04-26T05:04:46.500477+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:00:15.738485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":76,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.506386+0000","last_change":"2024-04-26T05:03:51.803993+0000","last_active":"2024-04-26T05:04:47.506386+0000","last_peered":"2024-04-26T05:04:47.506386+0000","last_clean":"2024-04-26T05:04:47.506386+0000","last_became_active":"2024-04-26T05:03:51.800058+0000","last_became_peered":"2024-04-26T05:03:51.800058+0000","last_unstale":"2024-04-26T05:04:47.506386+0000","last_undegraded":"2024-04-26T05:04:47.506386+0000","last_fullsized":"2024-04-26T05:04:47.506386+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:15:59.299215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":51,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.506978+0000","last_change":"2024-04-26T05:04:12.123623+0000","last_active":"2024-04-26T05:04:47.506978+0000","last_peered":"2024-04-26T05:04:47.506978+0000","last_clean":"2024-04-26T05:04:47.506978+0000","last_became_active":"2024-04-26T05:04:12.123284+0000","last_became_peered":"2024-04-26T05:04:12.123284+0000","last_unstale":"2024-04-26T05:04:47.506978+0000","last_undegraded":"2024-04-26T05:04:47.506978+0000","last_fullsized":"2024-04-26T05:04:47.506978+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:12:47.456887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":81,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.508791+0000","last_change":"2024-04-26T05:04:47.508791+0000","last_active":"2024-04-26T05:04:47.508791+0000","last_peered":"2024-04-26T05:04:47.508791+0000","last_clean":"2024-04-26T05:04:47.508791+0000","last_became_active":"2024-04-26T05:04:47.508464+0000","last_became_peered":"2024-04-26T05:04:47.508464+0000","last_unstale":"2024-04-26T05:04:47.508791+0000","last_undegraded":"2024-04-26T05:04:47.508791+0000","last_fullsized":"2024-04-26T05:04:47.508791+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:40:18.046251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":61,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.515543+0000","last_change":"2024-04-26T05:04:47.515543+0000","last_active":"2024-04-26T05:04:47.515543+0000","last_peered":"2024-04-26T05:04:47.515543+0000","last_clean":"2024-04-26T05:04:47.515543+0000","last_became_active":"2024-04-26T05:04:47.515146+0000","last_became_peered":"2024-04-26T05:04:47.515146+0000","last_unstale":"2024-04-26T05:04:47.515543+0000","last_undegraded":"2024-04-26T05:04:47.515543+0000","last_fullsized":"2024-04-26T05:04:47.515543+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:38:13.919522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":60,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.502562+0000","last_change":"2024-04-26T05:02:40.768748+0000","last_active":"2024-04-26T05:04:46.502562+0000","last_peered":"2024-04-26T05:04:46.502562+0000","last_clean":"2024-04-26T05:04:46.502562+0000","last_became_active":"2024-04-26T05:02:40.768413+0000","last_became_peered":"2024-04-26T05:02:40.768413+0000","last_unstale":"2024-04-26T05:04:46.502562+0000","last_undegraded":"2024-04-26T05:04:46.502562+0000","last_fullsized":"2024-04-26T05:04:46.502562+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:11:33.877754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":64,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.500596+0000","last_change":"2024-04-26T05:03:51.805183+0000","last_active":"2024-04-26T05:04:46.500596+0000","last_peered":"2024-04-26T05:04:46.500596+0000","last_clean":"2024-04-26T05:04:46.500596+0000","last_became_active":"2024-04-26T05:03:51.804789+0000","last_became_peered":"2024-04-26T05:03:51.804789+0000","last_unstale":"2024-04-26T05:04:46.500596+0000","last_undegraded":"2024-04-26T05:04:46.500596+0000","last_fullsized":"2024-04-26T05:04:46.500596+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:05:53.590354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":43,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.506077+0000","last_change":"2024-04-26T05:03:51.805254+0000","last_active":"2024-04-26T05:04:47.506077+0000","last_peered":"2024-04-26T05:04:47.506077+0000","last_clean":"2024-04-26T05:04:47.506077+0000","last_became_active":"2024-04-26T05:03:51.804923+0000","last_became_peered":"2024-04-26T05:03:51.804923+0000","last_unstale":"2024-04-26T05:04:47.506077+0000","last_undegraded":"2024-04-26T05:04:47.506077+0000","last_fullsized":"2024-04-26T05:04:47.506077+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:46:24.941436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":77,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.517100+0000","last_change":"2024-04-26T05:04:47.517100+0000","last_active":"2024-04-26T05:04:47.517100+0000","last_peered":"2024-04-26T05:04:47.517100+0000","last_clean":"2024-04-26T05:04:47.517100+0000","last_became_active":"2024-04-26T05:04:47.516646+0000","last_became_peered":"2024-04-26T05:04:47.516646+0000","last_unstale":"2024-04-26T05:04:47.517100+0000","last_undegraded":"2024-04-26T05:04:47.517100+0000","last_fullsized":"2024-04-26T05:04:47.517100+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:04:59.815879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":81,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.508261+0000","last_change":"2024-04-26T05:03:51.801036+0000","last_active":"2024-04-26T05:04:47.508261+0000","last_peered":"2024-04-26T05:04:47.508261+0000","last_clean":"2024-04-26T05:04:47.508261+0000","last_became_active":"2024-04-26T05:03:51.800724+0000","last_became_peered":"2024-04-26T05:03:51.800724+0000","last_unstale":"2024-04-26T05:04:47.508261+0000","last_undegraded":"2024-04-26T05:04:47.508261+0000","last_fullsized":"2024-04-26T05:04:47.508261+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:53:56.840151+0000","last_clean_scrub_stamp":"2024-04-26T04:53:56.840151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:09:11.757837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":52,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.516618+0000","last_change":"2024-04-26T05:04:47.516618+0000","last_active":"2024-04-26T05:04:47.516618+0000","last_peered":"2024-04-26T05:04:47.516618+0000","last_clean":"2024-04-26T05:04:47.516618+0000","last_became_active":"2024-04-26T05:04:47.516314+0000","last_became_peered":"2024-04-26T05:04:47.516314+0000","last_unstale":"2024-04-26T05:04:47.516618+0000","last_undegraded":"2024-04-26T05:04:47.516618+0000","last_fullsized":"2024-04-26T05:04:47.516618+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:27:25.679342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":63,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.517784+0000","last_change":"2024-04-26T05:04:47.517784+0000","last_active":"2024-04-26T05:04:47.517784+0000","last_peered":"2024-04-26T05:04:47.517784+0000","last_clean":"2024-04-26T05:04:47.517784+0000","last_became_active":"2024-04-26T05:04:47.517434+0000","last_became_peered":"2024-04-26T05:04:47.517434+0000","last_unstale":"2024-04-26T05:04:47.517784+0000","last_undegraded":"2024-04-26T05:04:47.517784+0000","last_fullsized":"2024-04-26T05:04:47.517784+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:08:54.244848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.504666+0000","last_change":"2024-04-26T05:03:51.806769+0000","last_active":"2024-04-26T05:04:46.504666+0000","last_peered":"2024-04-26T05:04:46.504666+0000","last_clean":"2024-04-26T05:04:46.504666+0000","last_became_active":"2024-04-26T05:03:51.806170+0000","last_became_peered":"2024-04-26T05:03:51.806170+0000","last_unstale":"2024-04-26T05:04:46.504666+0000","last_undegraded":"2024-04-26T05:04:46.504666+0000","last_fullsized":"2024-04-26T05:04:46.504666+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:21:39.768297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":60,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.501127+0000","last_change":"2024-04-26T05:04:12.125834+0000","last_active":"2024-04-26T05:04:46.501127+0000","last_peered":"2024-04-26T05:04:46.501127+0000","last_clean":"2024-04-26T05:04:46.501127+0000","last_became_active":"2024-04-26T05:04:12.125517+0000","last_became_peered":"2024-04-26T05:04:12.125517+0000","last_unstale":"2024-04-26T05:04:46.501127+0000","last_undegraded":"2024-04-26T05:04:46.501127+0000","last_fullsized":"2024-04-26T05:04:46.501127+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:13:16.391364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":76,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.505297+0000","last_change":"2024-04-26T05:04:12.125007+0000","last_active":"2024-04-26T05:04:46.505297+0000","last_peered":"2024-04-26T05:04:46.505297+0000","last_clean":"2024-04-26T05:04:46.505297+0000","last_became_active":"2024-04-26T05:04:12.123926+0000","last_became_peered":"2024-04-26T05:04:12.123926+0000","last_unstale":"2024-04-26T05:04:46.505297+0000","last_undegraded":"2024-04-26T05:04:46.505297+0000","last_fullsized":"2024-04-26T05:04:46.505297+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:24:39.761901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":60,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.504406+0000","last_change":"2024-04-26T05:03:51.805074+0000","last_active":"2024-04-26T05:04:46.504406+0000","last_peered":"2024-04-26T05:04:46.504406+0000","last_clean":"2024-04-26T05:04:46.504406+0000","last_became_active":"2024-04-26T05:03:51.804710+0000","last_became_peered":"2024-04-26T05:03:51.804710+0000","last_unstale":"2024-04-26T05:04:46.504406+0000","last_undegraded":"2024-04-26T05:04:46.504406+0000","last_fullsized":"2024-04-26T05:04:46.504406+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:48:49.853794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":59,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.523083+0000","last_change":"2024-04-26T05:04:47.523083+0000","last_active":"2024-04-26T05:04:47.523083+0000","last_peered":"2024-04-26T05:04:47.523083+0000","last_clean":"2024-04-26T05:04:47.523083+0000","last_became_active":"2024-04-26T05:04:47.522652+0000","last_became_peered":"2024-04-26T05:04:47.522652+0000","last_unstale":"2024-04-26T05:04:47.523083+0000","last_undegraded":"2024-04-26T05:04:47.523083+0000","last_fullsized":"2024-04-26T05:04:47.523083+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:05:30.198750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":60,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.514735+0000","last_change":"2024-04-26T05:04:47.514735+0000","last_active":"2024-04-26T05:04:47.514735+0000","last_peered":"2024-04-26T05:04:47.514735+0000","last_clean":"2024-04-26T05:04:47.514735+0000","last_became_active":"2024-04-26T05:04:47.514362+0000","last_became_peered":"2024-04-26T05:04:47.514362+0000","last_unstale":"2024-04-26T05:04:47.514735+0000","last_undegraded":"2024-04-26T05:04:47.514735+0000","last_fullsized":"2024-04-26T05:04:47.514735+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:41:30.261068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":63,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.501825+0000","last_change":"2024-04-26T05:03:51.801392+0000","last_active":"2024-04-26T05:04:47.501825+0000","last_peered":"2024-04-26T05:04:47.501825+0000","last_clean":"2024-04-26T05:04:47.501825+0000","last_became_active":"2024-04-26T05:03:51.800856+0000","last_became_peered":"2024-04-26T05:03:51.800856+0000","last_unstale":"2024-04-26T05:04:47.501825+0000","last_undegraded":"2024-04-26T05:04:47.501825+0000","last_fullsized":"2024-04-26T05:04:47.501825+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:05:58.425337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":60,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.521951+0000","last_change":"2024-04-26T05:04:47.521951+0000","last_active":"2024-04-26T05:04:47.521951+0000","last_peered":"2024-04-26T05:04:47.521951+0000","last_clean":"2024-04-26T05:04:47.521951+0000","last_became_active":"2024-04-26T05:04:47.521588+0000","last_became_peered":"2024-04-26T05:04:47.521588+0000","last_unstale":"2024-04-26T05:04:47.521951+0000","last_undegraded":"2024-04-26T05:04:47.521951+0000","last_fullsized":"2024-04-26T05:04:47.521951+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:04:26.827180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.504251+0000","last_change":"2024-04-26T05:03:51.803035+0000","last_active":"2024-04-26T05:04:46.504251+0000","last_peered":"2024-04-26T05:04:46.504251+0000","last_clean":"2024-04-26T05:04:46.504251+0000","last_became_active":"2024-04-26T05:03:51.802703+0000","last_became_peered":"2024-04-26T05:03:51.802703+0000","last_unstale":"2024-04-26T05:04:46.504251+0000","last_undegraded":"2024-04-26T05:04:46.504251+0000","last_fullsized":"2024-04-26T05:04:46.504251+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:01:22.472530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":43,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.498208+0000","last_change":"2024-04-26T04:55:07.451952+0000","last_active":"2024-04-26T05:04:46.498208+0000","last_peered":"2024-04-26T05:04:46.498208+0000","last_clean":"2024-04-26T05:04:46.498208+0000","last_became_active":"2024-04-26T04:55:07.451660+0000","last_became_peered":"2024-04-26T04:55:07.451660+0000","last_unstale":"2024-04-26T05:04:46.498208+0000","last_undegraded":"2024-04-26T05:04:46.498208+0000","last_fullsized":"2024-04-26T05:04:46.498208+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:36:14.683401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":76,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.518564+0000","last_change":"2024-04-26T05:04:47.518564+0000","last_active":"2024-04-26T05:04:47.518564+0000","last_peered":"2024-04-26T05:04:47.518564+0000","last_clean":"2024-04-26T05:04:47.518564+0000","last_became_active":"2024-04-26T05:04:47.518099+0000","last_became_peered":"2024-04-26T05:04:47.518099+0000","last_unstale":"2024-04-26T05:04:47.518564+0000","last_undegraded":"2024-04-26T05:04:47.518564+0000","last_fullsized":"2024-04-26T05:04:47.518564+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:00:23.156348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":61,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.498095+0000","last_change":"2024-04-26T05:03:51.806785+0000","last_active":"2024-04-26T05:04:46.498095+0000","last_peered":"2024-04-26T05:04:46.498095+0000","last_clean":"2024-04-26T05:04:46.498095+0000","last_became_active":"2024-04-26T05:03:51.806370+0000","last_became_peered":"2024-04-26T05:03:51.806370+0000","last_unstale":"2024-04-26T05:04:46.498095+0000","last_undegraded":"2024-04-26T05:04:46.498095+0000","last_fullsized":"2024-04-26T05:04:46.498095+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:51:41.841527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":42,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.500014+0000","last_change":"2024-04-26T04:55:07.500711+0000","last_active":"2024-04-26T05:04:46.500014+0000","last_peered":"2024-04-26T05:04:46.500014+0000","last_clean":"2024-04-26T05:04:46.500014+0000","last_became_active":"2024-04-26T04:55:07.500428+0000","last_became_peered":"2024-04-26T04:55:07.500428+0000","last_unstale":"2024-04-26T05:04:46.500014+0000","last_undegraded":"2024-04-26T05:04:46.500014+0000","last_fullsized":"2024-04-26T05:04:46.500014+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:07:04.979529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":60,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.498800+0000","last_change":"2024-04-26T05:02:40.768192+0000","last_active":"2024-04-26T05:04:46.498800+0000","last_peered":"2024-04-26T05:04:46.498800+0000","last_clean":"2024-04-26T05:04:46.498800+0000","last_became_active":"2024-04-26T05:02:40.767729+0000","last_became_peered":"2024-04-26T05:02:40.767729+0000","last_unstale":"2024-04-26T05:04:46.498800+0000","last_undegraded":"2024-04-26T05:04:46.498800+0000","last_fullsized":"2024-04-26T05:04:46.498800+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:42:52.054207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":76,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.519627+0000","last_change":"2024-04-26T05:04:47.519627+0000","last_active":"2024-04-26T05:04:47.519627+0000","last_peered":"2024-04-26T05:04:47.519627+0000","last_clean":"2024-04-26T05:04:47.519627+0000","last_became_active":"2024-04-26T05:04:47.518930+0000","last_became_peered":"2024-04-26T05:04:47.518930+0000","last_unstale":"2024-04-26T05:04:47.519627+0000","last_undegraded":"2024-04-26T05:04:47.519627+0000","last_fullsized":"2024-04-26T05:04:47.519627+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:16:01.807071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":68,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.502979+0000","last_change":"2024-04-26T05:02:40.768573+0000","last_active":"2024-04-26T05:04:46.502979+0000","last_peered":"2024-04-26T05:04:46.502979+0000","last_clean":"2024-04-26T05:04:46.502979+0000","last_became_active":"2024-04-26T05:02:40.768038+0000","last_became_peered":"2024-04-26T05:02:40.768038+0000","last_unstale":"2024-04-26T05:04:46.502979+0000","last_undegraded":"2024-04-26T05:04:46.502979+0000","last_fullsized":"2024-04-26T05:04:46.502979+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:17:09.118193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":63,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.518264+0000","last_change":"2024-04-26T05:04:47.517331+0000","last_active":"2024-04-26T05:04:47.518264+0000","last_peered":"2024-04-26T05:04:47.518264+0000","last_clean":"2024-04-26T05:04:47.518264+0000","last_became_active":"2024-04-26T05:04:47.517012+0000","last_became_peered":"2024-04-26T05:04:47.517012+0000","last_unstale":"2024-04-26T05:04:47.518264+0000","last_undegraded":"2024-04-26T05:04:47.518264+0000","last_fullsized":"2024-04-26T05:04:47.518264+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:17:54.745143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":42,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.497124+0000","last_change":"2024-04-26T04:55:07.452729+0000","last_active":"2024-04-26T05:04:46.497124+0000","last_peered":"2024-04-26T05:04:46.497124+0000","last_clean":"2024-04-26T05:04:46.497124+0000","last_became_active":"2024-04-26T04:55:07.452426+0000","last_became_peered":"2024-04-26T04:55:07.452426+0000","last_unstale":"2024-04-26T05:04:46.497124+0000","last_undegraded":"2024-04-26T05:04:46.497124+0000","last_fullsized":"2024-04-26T05:04:46.497124+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:59:48.509688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":59,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.517023+0000","last_change":"2024-04-26T05:04:47.517023+0000","last_active":"2024-04-26T05:04:47.517023+0000","last_peered":"2024-04-26T05:04:47.517023+0000","last_clean":"2024-04-26T05:04:47.517023+0000","last_became_active":"2024-04-26T05:04:47.516646+0000","last_became_peered":"2024-04-26T05:04:47.516646+0000","last_unstale":"2024-04-26T05:04:47.517023+0000","last_undegraded":"2024-04-26T05:04:47.517023+0000","last_fullsized":"2024-04-26T05:04:47.517023+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:09:28.891107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":62,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.510533+0000","last_change":"2024-04-26T05:04:47.510190+0000","last_active":"2024-04-26T05:04:47.510533+0000","last_peered":"2024-04-26T05:04:47.510533+0000","last_clean":"2024-04-26T05:04:47.510533+0000","last_became_active":"2024-04-26T05:04:47.509754+0000","last_became_peered":"2024-04-26T05:04:47.509754+0000","last_unstale":"2024-04-26T05:04:47.510533+0000","last_undegraded":"2024-04-26T05:04:47.510533+0000","last_fullsized":"2024-04-26T05:04:47.510533+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:21:14.228592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":62,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.518993+0000","last_change":"2024-04-26T05:04:47.518993+0000","last_active":"2024-04-26T05:04:47.518993+0000","last_peered":"2024-04-26T05:04:47.518993+0000","last_clean":"2024-04-26T05:04:47.518993+0000","last_became_active":"2024-04-26T05:04:47.518384+0000","last_became_peered":"2024-04-26T05:04:47.518384+0000","last_unstale":"2024-04-26T05:04:47.518993+0000","last_undegraded":"2024-04-26T05:04:47.518993+0000","last_fullsized":"2024-04-26T05:04:47.518993+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:27:00.437340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":61,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.497511+0000","last_change":"2024-04-26T05:04:12.124365+0000","last_active":"2024-04-26T05:04:46.497511+0000","last_peered":"2024-04-26T05:04:46.497511+0000","last_clean":"2024-04-26T05:04:46.497511+0000","last_became_active":"2024-04-26T05:04:12.123898+0000","last_became_peered":"2024-04-26T05:04:12.123898+0000","last_unstale":"2024-04-26T05:04:46.497511+0000","last_undegraded":"2024-04-26T05:04:46.497511+0000","last_fullsized":"2024-04-26T05:04:46.497511+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:04:45.820110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.503106+0000","last_change":"2024-04-26T05:03:51.802338+0000","last_active":"2024-04-26T05:04:46.503106+0000","last_peered":"2024-04-26T05:04:46.503106+0000","last_clean":"2024-04-26T05:04:46.503106+0000","last_became_active":"2024-04-26T05:03:51.802024+0000","last_became_peered":"2024-04-26T05:03:51.802024+0000","last_unstale":"2024-04-26T05:04:46.503106+0000","last_undegraded":"2024-04-26T05:04:46.503106+0000","last_fullsized":"2024-04-26T05:04:46.503106+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:08:37.594439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":65,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.505050+0000","last_change":"2024-04-26T04:55:07.451574+0000","last_active":"2024-04-26T05:04:46.505050+0000","last_peered":"2024-04-26T05:04:46.505050+0000","last_clean":"2024-04-26T05:04:46.505050+0000","last_became_active":"2024-04-26T04:55:07.451154+0000","last_became_peered":"2024-04-26T04:55:07.451154+0000","last_unstale":"2024-04-26T05:04:46.505050+0000","last_undegraded":"2024-04-26T05:04:46.505050+0000","last_fullsized":"2024-04-26T05:04:46.505050+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:56:04.497071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":56,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.500605+0000","last_change":"2024-04-26T05:04:12.126008+0000","last_active":"2024-04-26T05:04:46.500605+0000","last_peered":"2024-04-26T05:04:46.500605+0000","last_clean":"2024-04-26T05:04:46.500605+0000","last_became_active":"2024-04-26T05:04:12.125566+0000","last_became_peered":"2024-04-26T05:04:12.125566+0000","last_unstale":"2024-04-26T05:04:46.500605+0000","last_undegraded":"2024-04-26T05:04:46.500605+0000","last_fullsized":"2024-04-26T05:04:46.500605+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:32:57.921469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":62,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.518237+0000","last_change":"2024-04-26T05:04:47.518237+0000","last_active":"2024-04-26T05:04:47.518237+0000","last_peered":"2024-04-26T05:04:47.518237+0000","last_clean":"2024-04-26T05:04:47.518237+0000","last_became_active":"2024-04-26T05:04:47.517876+0000","last_became_peered":"2024-04-26T05:04:47.517876+0000","last_unstale":"2024-04-26T05:04:47.518237+0000","last_undegraded":"2024-04-26T05:04:47.518237+0000","last_fullsized":"2024-04-26T05:04:47.518237+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:21:16.591946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.523513+0000","last_change":"2024-04-26T05:04:47.523513+0000","last_active":"2024-04-26T05:04:47.523513+0000","last_peered":"2024-04-26T05:04:47.523513+0000","last_clean":"2024-04-26T05:04:47.523513+0000","last_became_active":"2024-04-26T05:04:47.523248+0000","last_became_peered":"2024-04-26T05:04:47.523248+0000","last_unstale":"2024-04-26T05:04:47.523513+0000","last_undegraded":"2024-04-26T05:04:47.523513+0000","last_fullsized":"2024-04-26T05:04:47.523513+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:28:42.069788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":60,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.500130+0000","last_change":"2024-04-26T05:03:51.805729+0000","last_active":"2024-04-26T05:04:46.500130+0000","last_peered":"2024-04-26T05:04:46.500130+0000","last_clean":"2024-04-26T05:04:46.500130+0000","last_became_active":"2024-04-26T05:03:51.805401+0000","last_became_peered":"2024-04-26T05:03:51.805401+0000","last_unstale":"2024-04-26T05:04:46.500130+0000","last_undegraded":"2024-04-26T05:04:46.500130+0000","last_fullsized":"2024-04-26T05:04:46.500130+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:05:36.172467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":62,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.521896+0000","last_change":"2024-04-26T05:04:47.521896+0000","last_active":"2024-04-26T05:04:47.521896+0000","last_peered":"2024-04-26T05:04:47.521896+0000","last_clean":"2024-04-26T05:04:47.521896+0000","last_became_active":"2024-04-26T05:04:47.521441+0000","last_became_peered":"2024-04-26T05:04:47.521441+0000","last_unstale":"2024-04-26T05:04:47.521896+0000","last_undegraded":"2024-04-26T05:04:47.521896+0000","last_fullsized":"2024-04-26T05:04:47.521896+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:47:15.628754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":62,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.517199+0000","last_change":"2024-04-26T05:04:47.516799+0000","last_active":"2024-04-26T05:04:47.517199+0000","last_peered":"2024-04-26T05:04:47.517199+0000","last_clean":"2024-04-26T05:04:47.517199+0000","last_became_active":"2024-04-26T05:04:47.516430+0000","last_became_peered":"2024-04-26T05:04:47.516430+0000","last_unstale":"2024-04-26T05:04:47.517199+0000","last_undegraded":"2024-04-26T05:04:47.517199+0000","last_fullsized":"2024-04-26T05:04:47.517199+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:34:03.727441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.505720+0000","last_change":"2024-04-26T05:03:51.800451+0000","last_active":"2024-04-26T05:04:46.505720+0000","last_peered":"2024-04-26T05:04:46.505720+0000","last_clean":"2024-04-26T05:04:46.505720+0000","last_became_active":"2024-04-26T05:03:51.800110+0000","last_became_peered":"2024-04-26T05:03:51.800110+0000","last_unstale":"2024-04-26T05:04:46.505720+0000","last_undegraded":"2024-04-26T05:04:46.505720+0000","last_fullsized":"2024-04-26T05:04:46.505720+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:03:47.250736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":80,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.513933+0000","last_change":"2024-04-26T05:04:47.513933+0000","last_active":"2024-04-26T05:04:47.513933+0000","last_peered":"2024-04-26T05:04:47.513933+0000","last_clean":"2024-04-26T05:04:47.513933+0000","last_became_active":"2024-04-26T05:04:47.512605+0000","last_became_peered":"2024-04-26T05:04:47.512605+0000","last_unstale":"2024-04-26T05:04:47.513933+0000","last_undegraded":"2024-04-26T05:04:47.513933+0000","last_fullsized":"2024-04-26T05:04:47.513933+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:56:26.131873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":63,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.517826+0000","last_change":"2024-04-26T05:04:47.517826+0000","last_active":"2024-04-26T05:04:47.517826+0000","last_peered":"2024-04-26T05:04:47.517826+0000","last_clean":"2024-04-26T05:04:47.517826+0000","last_became_active":"2024-04-26T05:04:47.517502+0000","last_became_peered":"2024-04-26T05:04:47.517502+0000","last_unstale":"2024-04-26T05:04:47.517826+0000","last_undegraded":"2024-04-26T05:04:47.517826+0000","last_fullsized":"2024-04-26T05:04:47.517826+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:18:34.533782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":51,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.513174+0000","last_change":"2024-04-26T05:04:47.513174+0000","last_active":"2024-04-26T05:04:47.513174+0000","last_peered":"2024-04-26T05:04:47.513174+0000","last_clean":"2024-04-26T05:04:47.513174+0000","last_became_active":"2024-04-26T05:04:47.512833+0000","last_became_peered":"2024-04-26T05:04:47.512833+0000","last_unstale":"2024-04-26T05:04:47.513174+0000","last_undegraded":"2024-04-26T05:04:47.513174+0000","last_fullsized":"2024-04-26T05:04:47.513174+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:16:42.884641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":62,"reported_epoch":100,"state":"active+clean","last_fresh":"2024-04-26T05:04:47.514646+0000","last_change":"2024-04-26T05:04:47.514646+0000","last_active":"2024-04-26T05:04:47.514646+0000","last_peered":"2024-04-26T05:04:47.514646+0000","last_clean":"2024-04-26T05:04:47.514646+0000","last_became_active":"2024-04-26T05:04:47.514291+0000","last_became_peered":"2024-04-26T05:04:47.514291+0000","last_unstale":"2024-04-26T05:04:47.514646+0000","last_undegraded":"2024-04-26T05:04:47.514646+0000","last_fullsized":"2024-04-26T05:04:47.514646+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:07.435022+0000","last_clean_scrub_stamp":"2024-04-26T04:55:07.435022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:01:50.135758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":78,"reported_epoch":99,"state":"active+clean","last_fresh":"2024-04-26T05:04:46.501039+0000","last_change":"2024-04-26T05:04:12.124882+0000","last_active":"2024-04-26T05:04:46.501039+0000","last_peered":"2024-04-26T05:04:46.501039+0000","last_clean":"2024-04-26T05:04:46.501039+0000","last_became_active":"2024-04-26T05:04:12.124468+0000","last_became_peered":"2024-04-26T05:04:12.124468+0000","last_unstale":"2024-04-26T05:04:46.501039+0000","last_undegraded":"2024-04-26T05:04:46.501039+0000","last_fullsized":"2024-04-26T05:04:46.501039+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:55:06.433485+0000","last_clean_scrub_stamp":"2024-04-26T04:55:06.433485+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:30:14.534509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1053,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201431,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311308,"kb_used_data":1664,"kb_used_omap":1,"kb_used_meta":309630,"kb_avail":93433844,"statfs":{"total":95995035648,"available":95676256256,"internally_reserved":0,"allocated":1703936,"data_stored":1151251,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317061464},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463068,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299012,"kb_used_data":1648,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446140,"statfs":{"total":95995035648,"available":95688847360,"internally_reserved":0,"allocated":1687552,"data_stored":1150890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":89,"seq":382252089369,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28592,"kb_used_data":1060,"kb_used_omap":8,"kb_used_meta":27511,"kb_avail":93716560,"statfs":{"total":95995035648,"available":95965757440,"internally_reserved":0,"allocated":1085440,"data_stored":556274,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8667,"internal_metadata":28171813},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":84,"seq":360777252903,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":29176,"kb_used_data":1644,"kb_used_omap":18,"kb_used_meta":27501,"kb_avail":93715976,"statfs":{"total":95995035648,"available":95965159424,"internally_reserved":0,"allocated":1683456,"data_stored":1146969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19374,"internal_metadata":28161106},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":94,"seq":403726925845,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28604,"kb_used_data":1068,"kb_used_omap":47,"kb_used_meta":27472,"kb_avail":93716548,"statfs":{"total":95995035648,"available":95965745152,"internally_reserved":0,"allocated":1093632,"data_stored":556878,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49111,"internal_metadata":28131369},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":99,"seq":425201762318,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28596,"kb_used_data":1064,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93716556,"statfs":{"total":95995035648,"available":95965753344,"internally_reserved":0,"allocated":1089536,"data_stored":556356,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7696,"internal_metadata":28172784},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986341,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302536,"kb_used_data":1084,"kb_used_omap":1,"kb_used_meta":301438,"kb_avail":93442616,"statfs":{"total":95995035648,"available":95685238784,"internally_reserved":0,"allocated":1110016,"data_stored":561136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308672856},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724705,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306620,"kb_used_data":1064,"kb_used_omap":1,"kb_used_meta":305534,"kb_avail":93438532,"statfs":{"total":95995035648,"available":95681056768,"internally_reserved":0,"allocated":1089536,"data_stored":556563,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312867160},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1053,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-26T05:05:47.841 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-26T05:05:47.841 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-26T05:05:47.841 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-26T05:05:47.842 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:05:48.113 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:48.236 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:05:47 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:05:47] "GET /metrics HTTP/1.1" 200 34019 "" "Prometheus/2.43.0" 2024-04-26T05:05:48.931 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:48 smithi012 ceph-mon[91852]: from='client.44486 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T05:05:48.931 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:48 smithi012 ceph-mon[91852]: pgmap v233: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:48.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:48 smithi012 ceph-mon[95978]: from='client.44486 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T05:05:48.931 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:48 smithi012 ceph-mon[95978]: pgmap v233: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:48.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:48 smithi116 ceph-mon[85502]: from='client.44486 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T05:05:48.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:48 smithi116 ceph-mon[85502]: pgmap v233: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:50.067 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:05:50.067 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:05:50.719 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:50 smithi012 ceph-mon[91852]: pgmap v234: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:50.719 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:50 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/281830357' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:50.719 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:50 smithi012 ceph-mon[95978]: pgmap v234: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:50.719 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:50 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/281830357' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:50.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:50 smithi116 ceph-mon[85502]: pgmap v234: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:50.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:50 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/281830357' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:52.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:52 smithi116 ceph-mon[85502]: pgmap v235: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:53.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:52 smithi012 ceph-mon[95978]: pgmap v235: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:53.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:52 smithi012 ceph-mon[91852]: pgmap v235: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:53.722 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:05:53.999 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:54.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:54 smithi012 ceph-mon[91852]: pgmap v236: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:54.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:54 smithi012 ceph-mon[95978]: pgmap v236: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:54.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:54 smithi116 ceph-mon[85502]: pgmap v236: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:55.901 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:55 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:05:55.902 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:05:55.902 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:05:55.902 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:55 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:05:55.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:55 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:05:56.971 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:56 smithi012 ceph-mon[91852]: pgmap v237: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:56.971 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:56 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/3066565799' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:56.971 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:56 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:56.972 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:56 smithi012 ceph-mon[95978]: pgmap v237: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:56.972 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:56 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/3066565799' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:56.972 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:56 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:56.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:56 smithi116 ceph-mon[85502]: pgmap v237: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:56.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:56 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/3066565799' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:56.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:56 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:58.045 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:05:57 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:05:57] "GET /metrics HTTP/1.1" 200 34018 "" "Prometheus/2.43.0" 2024-04-26T05:05:58.782 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:05:58 smithi012 ceph-mon[91852]: pgmap v238: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:58.782 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:05:58 smithi012 ceph-mon[95978]: pgmap v238: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:58.894 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:05:58 smithi116 ceph-mon[85502]: pgmap v238: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:05:59.506 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:05:59.787 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:00.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:00 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:06:00.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:00 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:06:00.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:00 smithi012 ceph-mon[91852]: pgmap v239: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:00.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:00 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:06:00.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:00 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:06:00.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:00 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:06:00.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:00 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:06:00.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:00 smithi012 ceph-mon[95978]: pgmap v239: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:00.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:00 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:06:00.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:00 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:06:00.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:00 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:06:00.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:00 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:06:00.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:00 smithi116 ceph-mon[85502]: pgmap v239: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:00.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:00 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:06:00.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:00 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:06:01.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:01 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:06:01.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:01 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:06:01.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:01 smithi012 ceph-mon[91852]: pgmap v240: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:01.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:01 smithi012 ceph-mon[91852]: pgmap v241: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:01.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:01 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:06:01.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:01 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:06:01.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:01 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:06:01.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:01 smithi012 ceph-mon[95978]: pgmap v240: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:01.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:01 smithi012 ceph-mon[95978]: pgmap v241: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:01.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:01 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:06:01.670 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:06:01.671 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:06:01.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:01 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:06:01.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:01 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:06:01.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:01 smithi116 ceph-mon[85502]: pgmap v240: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:01.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:01 smithi116 ceph-mon[85502]: pgmap v241: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:01.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:01 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:06:02.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:02 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/3300596618' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:02.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:02 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/3300596618' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:02.657 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:02 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/3300596618' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:03.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:03 smithi116 ceph-mon[85502]: pgmap v242: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:03.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:03 smithi012 ceph-mon[91852]: pgmap v242: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:03.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:03 smithi012 ceph-mon[95978]: pgmap v242: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:05.370 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:06:05.647 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:05.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:05 smithi012 ceph-mon[91852]: pgmap v243: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:05.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:05 smithi012 ceph-mon[95978]: pgmap v243: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:05.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:05 smithi116 ceph-mon[85502]: pgmap v243: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:07.460 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:06:07.460 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:06:07.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:07 smithi012 ceph-mon[91852]: pgmap v244: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:07.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:07 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/960679608' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:07.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:07 smithi012 ceph-mon[95978]: pgmap v244: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:07.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:07 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/960679608' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:07.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:07 smithi116 ceph-mon[85502]: pgmap v244: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:07.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:07 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/960679608' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:08.109 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:06:07 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:06:07] "GET /metrics HTTP/1.1" 200 34018 "" "Prometheus/2.43.0" 2024-04-26T05:06:09.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:09 smithi116 ceph-mon[85502]: pgmap v245: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:10.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:09 smithi012 ceph-mon[95978]: pgmap v245: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:10.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:09 smithi012 ceph-mon[91852]: pgmap v245: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:11.136 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:06:11.414 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:11.862 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:11 smithi012 ceph-mon[91852]: pgmap v246: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:11.862 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:11 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:06:11.862 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:11 smithi012 ceph-mon[95978]: pgmap v246: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:11.862 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:11 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:06:11.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:11 smithi116 ceph-mon[85502]: pgmap v246: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:11.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:11 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:06:13.324 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:06:13.325 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:06:13.898 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:13 smithi012 ceph-mon[91852]: pgmap v247: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:13.899 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:13 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/759070317' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:13.899 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:13 smithi012 ceph-mon[95978]: pgmap v247: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:13.899 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:13 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/759070317' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:13.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:13 smithi116 ceph-mon[85502]: pgmap v247: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:13.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:13 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/759070317' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:15.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:15 smithi116 ceph-mon[85502]: pgmap v248: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:16.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:15 smithi012 ceph-mon[91852]: pgmap v248: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:16.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:15 smithi012 ceph-mon[95978]: pgmap v248: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:16.925 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:06:17.205 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:17.991 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:17 smithi012 ceph-mon[91852]: pgmap v249: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:17.991 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:17 smithi012 ceph-mon[95978]: pgmap v249: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:17.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:17 smithi116 ceph-mon[85502]: pgmap v249: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:18.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:06:17 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:06:17] "GET /metrics HTTP/1.1" 200 34022 "" "Prometheus/2.43.0" 2024-04-26T05:06:19.045 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:06:19.045 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:06:19.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:19 smithi116 ceph-mon[85502]: pgmap v250: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:19.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:19 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/3651627139' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:20.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:19 smithi012 ceph-mon[91852]: pgmap v250: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:20.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:19 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/3651627139' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:20.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:19 smithi012 ceph-mon[95978]: pgmap v250: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:20.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:19 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/3651627139' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:21.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:21 smithi116 ceph-mon[85502]: pgmap v251: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:22.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:21 smithi012 ceph-mon[91852]: pgmap v251: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:22.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:21 smithi012 ceph-mon[95978]: pgmap v251: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:22.708 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:06:22.984 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:23.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:23 smithi116 ceph-mon[85502]: pgmap v252: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:24.011 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:23 smithi012 ceph-mon[91852]: pgmap v252: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:24.011 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:23 smithi012 ceph-mon[95978]: pgmap v252: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:24.793 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:06:24.793 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:06:25.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:25 smithi116 ceph-mon[85502]: pgmap v253: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:25.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:25 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/2006273240' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:26.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:25 smithi012 ceph-mon[91852]: pgmap v253: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:26.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:25 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/2006273240' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:26.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:25 smithi012 ceph-mon[95978]: pgmap v253: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:26.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:25 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/2006273240' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:26.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:26 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:06:27.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:26 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:06:27.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:26 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:06:27.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:27 smithi116 ceph-mon[85502]: pgmap v254: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:27.992 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:27 smithi012 ceph-mon[91852]: pgmap v254: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:27.992 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:27 smithi012 ceph-mon[95978]: pgmap v254: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:28.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:06:27 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:06:27] "GET /metrics HTTP/1.1" 200 34017 "" "Prometheus/2.43.0" 2024-04-26T05:06:28.389 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:06:28.664 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:29.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:29 smithi116 ceph-mon[85502]: pgmap v255: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:30.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:29 smithi012 ceph-mon[95978]: pgmap v255: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:30.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:29 smithi012 ceph-mon[91852]: pgmap v255: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:30.688 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:06:30.688 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:06:31.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:31 smithi116 ceph-mon[85502]: pgmap v256: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:31.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:31 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/1229453935' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:32.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:31 smithi012 ceph-mon[91852]: pgmap v256: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:32.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:31 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/1229453935' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:32.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:31 smithi012 ceph-mon[95978]: pgmap v256: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:32.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:31 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/1229453935' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:33.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:33 smithi116 ceph-mon[85502]: pgmap v257: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:34.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:33 smithi012 ceph-mon[91852]: pgmap v257: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:34.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:33 smithi012 ceph-mon[95978]: pgmap v257: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:34.421 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:06:34.699 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:35.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:35 smithi012 ceph-mon[91852]: pgmap v258: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:35.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:35 smithi012 ceph-mon[95978]: pgmap v258: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:35.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:35 smithi116 ceph-mon[85502]: pgmap v258: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:36.619 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:06:36.619 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:06:36.914 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:36 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/1367934905' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:36.914 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:36 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/1367934905' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:36.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:36 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/1367934905' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:37.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:37 smithi116 ceph-mon[85502]: pgmap v259: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:37.992 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:37 smithi012 ceph-mon[91852]: pgmap v259: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:37.992 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:37 smithi012 ceph-mon[95978]: pgmap v259: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:38.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:06:37 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:06:37] "GET /metrics HTTP/1.1" 200 34017 "" "Prometheus/2.43.0" 2024-04-26T05:06:39.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:39 smithi116 ceph-mon[85502]: pgmap v260: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:40.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:39 smithi012 ceph-mon[91852]: pgmap v260: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:40.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:39 smithi012 ceph-mon[95978]: pgmap v260: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:40.319 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:06:40.597 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:41.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:41 smithi116 ceph-mon[85502]: pgmap v261: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:41.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:41 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:06:42.007 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:41 smithi012 ceph-mon[91852]: pgmap v261: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:42.008 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:41 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:06:42.008 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:41 smithi012 ceph-mon[95978]: pgmap v261: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:42.008 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:41 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:06:42.525 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:06:42.525 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:06:42.871 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:42 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/3878548356' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:42.871 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:42 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/3878548356' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:42.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:42 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/3878548356' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:43.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:43 smithi116 ceph-mon[85502]: pgmap v262: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:44.012 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:43 smithi012 ceph-mon[95978]: pgmap v262: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:44.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:43 smithi012 ceph-mon[91852]: pgmap v262: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:45.994 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:45 smithi116 ceph-mon[85502]: pgmap v263: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:46.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:45 smithi012 ceph-mon[95978]: pgmap v263: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:46.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:45 smithi012 ceph-mon[91852]: pgmap v263: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:46.188 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:06:46.466 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:47.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:47 smithi012 ceph-mon[91852]: pgmap v264: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:47.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:47 smithi012 ceph-mon[95978]: pgmap v264: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:47.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:47 smithi116 ceph-mon[85502]: pgmap v264: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:48.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:06:47 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:06:47] "GET /metrics HTTP/1.1" 200 34021 "" "Prometheus/2.43.0" 2024-04-26T05:06:48.285 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:06:48.285 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:06:48.886 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:48 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/3353405279' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:48.887 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:48 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/3353405279' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:48.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:48 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/3353405279' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:49.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:49 smithi116 ceph-mon[85502]: pgmap v265: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:50.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:49 smithi012 ceph-mon[91852]: pgmap v265: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:50.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:49 smithi012 ceph-mon[95978]: pgmap v265: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:51.913 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:06:51.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:51 smithi116 ceph-mon[85502]: pgmap v266: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:52.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:51 smithi012 ceph-mon[91852]: pgmap v266: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:52.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:51 smithi012 ceph-mon[95978]: pgmap v266: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:52.193 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:53.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:53 smithi012 ceph-mon[91852]: pgmap v267: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:53.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:53 smithi012 ceph-mon[95978]: pgmap v267: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:53.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:53 smithi116 ceph-mon[85502]: pgmap v267: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:54.135 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:06:54.135 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:06:54.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:54 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/1543406215' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:55.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:54 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/1543406215' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:55.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:54 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/1543406215' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:06:55.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:55 smithi116 ceph-mon[85502]: pgmap v268: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:56.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:55 smithi012 ceph-mon[91852]: pgmap v268: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:56.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:55 smithi012 ceph-mon[95978]: pgmap v268: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:56.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:56 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:06:57.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:56 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:06:57.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:56 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:06:57.748 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:06:57.991 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:57 smithi012 ceph-mon[91852]: pgmap v269: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:57.991 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:57 smithi012 ceph-mon[95978]: pgmap v269: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:57.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:57 smithi116 ceph-mon[85502]: pgmap v269: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:58.026 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:06:58.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:06:57 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:06:57] "GET /metrics HTTP/1.1" 200 34020 "" "Prometheus/2.43.0" 2024-04-26T05:06:59.877 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:06:59.877 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:06:59.878 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:06:59 smithi012 ceph-mon[91852]: pgmap v270: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:59.878 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:06:59 smithi012 ceph-mon[95978]: pgmap v270: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:06:59.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:06:59 smithi116 ceph-mon[85502]: pgmap v270: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:01.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:00 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/4254853980' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:01.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:00 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:07:01.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:00 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/4254853980' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:01.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:00 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:07:01.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:00 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/4254853980' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:01.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:00 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:07:01.792 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:01 smithi012 ceph-mon[91852]: pgmap v271: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:01.793 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:01 smithi012 ceph-mon[95978]: pgmap v271: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:01.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:01 smithi116 ceph-mon[85502]: pgmap v271: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:03.473 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:07:04.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:03 smithi012 ceph-mon[91852]: pgmap v272: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:04.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:03 smithi012 ceph-mon[95978]: pgmap v272: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:04.060 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:03 smithi116 ceph-mon[85502]: pgmap v272: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:04.345 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:07:05.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:05 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:07:05.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:05 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:07:05.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:05 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:07:05.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:05 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:07:05.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:05 smithi012 ceph-mon[91852]: pgmap v273: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:05.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:05 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:07:05.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:05 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:07:05.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:05 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:07:05.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:05 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:07:05.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:05 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:07:05.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:05 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:07:05.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:05 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:07:05.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:05 smithi012 ceph-mon[95978]: pgmap v273: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:05.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:05 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:07:05.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:05 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:07:05.514 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:05 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:07:05.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:05 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:07:05.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:05 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:07:05.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:05 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:07:05.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:05 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:07:05.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:05 smithi116 ceph-mon[85502]: pgmap v273: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:05.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:05 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:07:05.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:05 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:07:05.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:05 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:07:06.369 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:07:06.369 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:07:06.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:06 smithi116 ceph-mon[85502]: pgmap v274: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:06.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:06 smithi116 ceph-mon[85502]: pgmap v275: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:06.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:06 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/2115632626' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:06.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:06 smithi012 ceph-mon[91852]: pgmap v274: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:06.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:06 smithi012 ceph-mon[91852]: pgmap v275: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:06.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:06 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/2115632626' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:06.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:06 smithi012 ceph-mon[95978]: pgmap v274: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:06.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:06 smithi012 ceph-mon[95978]: pgmap v275: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:06.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:06 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/2115632626' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:08.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:07:07 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:07:07] "GET /metrics HTTP/1.1" 200 34020 "" "Prometheus/2.43.0" 2024-04-26T05:07:08.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:08 smithi116 ceph-mon[85502]: pgmap v276: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:08.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:08 smithi012 ceph-mon[91852]: pgmap v276: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:08.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:08 smithi012 ceph-mon[95978]: pgmap v276: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:10.007 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:07:10.286 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:07:10.713 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:10 smithi012 ceph-mon[91852]: pgmap v277: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:10.714 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:10 smithi012 ceph-mon[95978]: pgmap v277: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:10.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:10 smithi116 ceph-mon[85502]: pgmap v277: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:11.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:11 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:07:11.749 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:11 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:07:11.749 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:11 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:07:12.239 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:07:12.239 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:07:12.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:12 smithi012 ceph-mon[91852]: pgmap v278: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:12.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:12 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/47190144' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:12.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:12 smithi012 ceph-mon[95978]: pgmap v278: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:12.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:12 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/47190144' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:12.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:12 smithi116 ceph-mon[85502]: pgmap v278: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:12.742 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:12 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/47190144' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:14.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:14 smithi116 ceph-mon[85502]: pgmap v279: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:14.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:14 smithi012 ceph-mon[91852]: pgmap v279: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:14.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:14 smithi012 ceph-mon[95978]: pgmap v279: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:15.890 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:07:16.167 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:07:16.628 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:16 smithi012 ceph-mon[91852]: pgmap v280: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:16.629 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:16 smithi012 ceph-mon[95978]: pgmap v280: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:16.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:16 smithi116 ceph-mon[85502]: pgmap v280: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:18.063 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:07:18.063 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:07:18.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:07:17 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:07:17] "GET /metrics HTTP/1.1" 200 34021 "" "Prometheus/2.43.0" 2024-04-26T05:07:18.692 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:18 smithi012 ceph-mon[91852]: pgmap v281: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:18.692 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:18 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/1040126673' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:18.692 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:18 smithi012 ceph-mon[95978]: pgmap v281: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:18.692 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:18 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/1040126673' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:18.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:18 smithi116 ceph-mon[85502]: pgmap v281: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:18.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:18 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/1040126673' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:20.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:20 smithi012 ceph-mon[91852]: pgmap v282: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:20.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:20 smithi012 ceph-mon[95978]: pgmap v282: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:20.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:20 smithi116 ceph-mon[85502]: pgmap v282: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:21.721 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:07:21.990 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:07:22.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:22 smithi012 ceph-mon[91852]: pgmap v283: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:22.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:22 smithi012 ceph-mon[95978]: pgmap v283: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:22.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:22 smithi116 ceph-mon[85502]: pgmap v283: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:23.943 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:07:23.943 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:07:24.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:24 smithi012 ceph-mon[91852]: pgmap v284: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:24.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:24 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/2354724843' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:24.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:24 smithi012 ceph-mon[95978]: pgmap v284: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:24.513 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:24 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/2354724843' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:24.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:24 smithi116 ceph-mon[85502]: pgmap v284: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:24.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:24 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/2354724843' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:26.502 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:26 smithi012 ceph-mon[91852]: pgmap v285: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:26.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:26 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:07:26.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:26 smithi012 ceph-mon[95978]: pgmap v285: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:26.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:26 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:07:26.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:26 smithi116 ceph-mon[85502]: pgmap v285: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:26.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:26 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:07:27.660 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:07:27.938 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:07:28.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:07:27 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:07:27] "GET /metrics HTTP/1.1" 200 34023 "" "Prometheus/2.43.0" 2024-04-26T05:07:28.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:28 smithi012 ceph-mon[91852]: pgmap v286: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:28.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:28 smithi012 ceph-mon[95978]: pgmap v286: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:28.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:28 smithi116 ceph-mon[85502]: pgmap v286: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:29.776 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:07:29.776 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:07:30.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:30 smithi012 ceph-mon[91852]: pgmap v287: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:30.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:30 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/3901744518' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:30.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:30 smithi012 ceph-mon[95978]: pgmap v287: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:30.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:30 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/3901744518' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:30.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:30 smithi116 ceph-mon[85502]: pgmap v287: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:30.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:30 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/3901744518' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:32.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:32 smithi012 ceph-mon[91852]: pgmap v288: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:32.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:32 smithi012 ceph-mon[95978]: pgmap v288: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:32.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:32 smithi116 ceph-mon[85502]: pgmap v288: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:33.492 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:07:33.769 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:07:34.794 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:34 smithi116 ceph-mon[85502]: pgmap v289: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:34.892 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:34 smithi012 ceph-mon[91852]: pgmap v289: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:34.892 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:34 smithi012 ceph-mon[95978]: pgmap v289: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:35.698 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:07:35.699 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:07:36.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:36 smithi012 ceph-mon[95978]: pgmap v290: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:36.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:36 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/1771718006' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:36.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:36 smithi012 ceph-mon[91852]: pgmap v290: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:36.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:36 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/1771718006' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:36.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:36 smithi116 ceph-mon[85502]: pgmap v290: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:36.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:36 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/1771718006' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:38.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:07:37 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:07:37] "GET /metrics HTTP/1.1" 200 34023 "" "Prometheus/2.43.0" 2024-04-26T05:07:38.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:38 smithi116 ceph-mon[85502]: pgmap v291: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:39.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:38 smithi012 ceph-mon[91852]: pgmap v291: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:39.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:38 smithi012 ceph-mon[95978]: pgmap v291: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:39.381 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:07:39.659 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:07:40.815 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:40 smithi012 ceph-mon[91852]: pgmap v292: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:40.816 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:40 smithi012 ceph-mon[95978]: pgmap v292: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:40.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:40 smithi116 ceph-mon[85502]: pgmap v292: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:41.631 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:07:41.631 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:07:41.926 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:41 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:07:41.927 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:41 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:07:41.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:41 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:07:42.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:42 smithi012 ceph-mon[91852]: pgmap v293: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:42.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:42 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/217385640' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:42.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:42 smithi012 ceph-mon[95978]: pgmap v293: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:42.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:42 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/217385640' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:42.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:42 smithi116 ceph-mon[85502]: pgmap v293: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:42.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:42 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/217385640' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:44.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:44 smithi116 ceph-mon[85502]: pgmap v294: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:45.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:44 smithi012 ceph-mon[91852]: pgmap v294: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:45.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:44 smithi012 ceph-mon[95978]: pgmap v294: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:45.322 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:07:45.600 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:07:46.829 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:46 smithi012 ceph-mon[91852]: pgmap v295: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:46.829 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:46 smithi012 ceph-mon[95978]: pgmap v295: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:46.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:46 smithi116 ceph-mon[85502]: pgmap v295: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:47.415 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:07:47.415 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:07:47.738 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:47 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/253015766' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:47.739 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:47 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/253015766' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:47.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:47 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/253015766' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:48.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:07:47 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:07:47] "GET /metrics HTTP/1.1" 200 34013 "" "Prometheus/2.43.0" 2024-04-26T05:07:48.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:48 smithi116 ceph-mon[85502]: pgmap v296: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:49.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:48 smithi012 ceph-mon[95978]: pgmap v296: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:49.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:48 smithi012 ceph-mon[91852]: pgmap v296: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:50.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:50 smithi116 ceph-mon[85502]: pgmap v297: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:51.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:50 smithi012 ceph-mon[95978]: pgmap v297: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:51.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:50 smithi012 ceph-mon[91852]: pgmap v297: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:51.021 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:07:51.298 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:07:52.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:52 smithi012 ceph-mon[91852]: pgmap v298: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:52.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:52 smithi012 ceph-mon[95978]: pgmap v298: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:52.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:52 smithi116 ceph-mon[85502]: pgmap v298: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:53.274 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:07:53.274 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:07:53.898 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:53 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/1511773628' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:53.898 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:53 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/1511773628' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:53.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:53 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/1511773628' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:54.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:54 smithi116 ceph-mon[85502]: pgmap v299: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:55.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:54 smithi012 ceph-mon[91852]: pgmap v299: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:55.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:54 smithi012 ceph-mon[95978]: pgmap v299: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:56.929 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:07:56.956 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:56 smithi012 ceph-mon[91852]: pgmap v300: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:56.956 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:56 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:07:56.957 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:56 smithi012 ceph-mon[95978]: pgmap v300: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:56.957 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:56 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:07:56.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:56 smithi116 ceph-mon[85502]: pgmap v300: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:56.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:56 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:07:57.208 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:07:58.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:07:57 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:07:57] "GET /metrics HTTP/1.1" 200 34020 "" "Prometheus/2.43.0" 2024-04-26T05:07:58.872 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:07:58.873 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:07:58.873 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:58 smithi012 ceph-mon[91852]: pgmap v301: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:58.873 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:58 smithi012 ceph-mon[95978]: pgmap v301: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:58.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:58 smithi116 ceph-mon[85502]: pgmap v301: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:07:59.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:07:59 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/196523557' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:59.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:07:59 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/196523557' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:07:59.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:07:59 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/196523557' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:00.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:00 smithi116 ceph-mon[85502]: pgmap v302: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:01.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:00 smithi012 ceph-mon[91852]: pgmap v302: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:01.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:00 smithi012 ceph-mon[95978]: pgmap v302: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:02.499 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:08:02.776 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:08:02.899 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:02 smithi012 ceph-mon[91852]: pgmap v303: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:02.899 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:02 smithi012 ceph-mon[95978]: pgmap v303: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:02.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:02 smithi116 ceph-mon[85502]: pgmap v303: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:04.542 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:08:04.542 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:08:04.963 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:04 smithi012 ceph-mon[91852]: pgmap v304: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:04.963 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:04 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/1480199175' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:04.963 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:04 smithi012 ceph-mon[95978]: pgmap v304: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:04.963 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:04 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/1480199175' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:04.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:04 smithi116 ceph-mon[85502]: pgmap v304: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:04.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:04 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/1480199175' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:05.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:05 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:08:06.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:05 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:08:06.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:05 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:08:06.861 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:06 smithi116 ceph-mon[85502]: pgmap v305: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:07.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:06 smithi012 ceph-mon[91852]: pgmap v305: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:07.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:06 smithi012 ceph-mon[95978]: pgmap v305: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:08.171 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:08:08.195 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:08:07 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:08:07] "GET /metrics HTTP/1.1" 200 34020 "" "Prometheus/2.43.0" 2024-04-26T05:08:08.912 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:08:08.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:08 smithi116 ceph-mon[85502]: pgmap v306: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:09.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:08 smithi012 ceph-mon[91852]: pgmap v306: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:09.012 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:08 smithi012 ceph-mon[95978]: pgmap v306: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:10.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:09 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:08:10.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:09 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:08:10.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:09 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:08:10.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:09 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:08:10.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:09 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:08:10.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:09 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:08:10.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:09 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:08:10.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:09 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:08:10.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:09 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:08:10.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:09 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:08:10.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:09 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:08:10.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:09 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:08:10.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:09 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:08:10.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:09 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:08:10.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:09 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:08:10.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:09 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:08:10.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:09 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:08:10.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:09 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:08:10.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:09 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:08:10.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:09 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:08:10.264 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:09 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:08:10.736 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:08:10.736 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:08:11.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:10 smithi012 ceph-mon[95978]: pgmap v307: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:11.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:10 smithi012 ceph-mon[95978]: pgmap v308: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:11.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:10 smithi012 ceph-mon[95978]: pgmap v309: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:11.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:10 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/4209236026' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:11.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:10 smithi012 ceph-mon[91852]: pgmap v307: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:11.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:10 smithi012 ceph-mon[91852]: pgmap v308: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:11.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:10 smithi012 ceph-mon[91852]: pgmap v309: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:11.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:10 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/4209236026' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:11.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:10 smithi116 ceph-mon[85502]: pgmap v307: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:11.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:10 smithi116 ceph-mon[85502]: pgmap v308: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:11.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:10 smithi116 ceph-mon[85502]: pgmap v309: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:11.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:10 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/4209236026' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:12.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:11 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:08:12.262 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:11 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:08:12.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:11 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:08:13.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:12 smithi116 ceph-mon[85502]: pgmap v310: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:13.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:12 smithi012 ceph-mon[91852]: pgmap v310: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:13.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:12 smithi012 ceph-mon[95978]: pgmap v310: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:14.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:13 smithi012 ceph-mon[91852]: pgmap v311: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:14.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:13 smithi012 ceph-mon[95978]: pgmap v311: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:14.357 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:08:14.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:13 smithi116 ceph-mon[85502]: pgmap v311: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:14.633 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:08:16.342 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:08:16.342 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:08:17.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:16 smithi116 ceph-mon[85502]: pgmap v312: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:17.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:16 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/1918105012' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:17.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:16 smithi012 ceph-mon[91852]: pgmap v312: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:17.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:16 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/1918105012' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:17.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:16 smithi012 ceph-mon[95978]: pgmap v312: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:17.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:16 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/1918105012' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:18.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:08:17 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:08:17] "GET /metrics HTTP/1.1" 200 34020 "" "Prometheus/2.43.0" 2024-04-26T05:08:19.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:18 smithi116 ceph-mon[85502]: pgmap v313: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:19.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:18 smithi012 ceph-mon[91852]: pgmap v313: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:19.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:18 smithi012 ceph-mon[95978]: pgmap v313: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:19.931 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:08:20.202 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:08:21.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:20 smithi012 ceph-mon[91852]: pgmap v314: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:21.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:20 smithi012 ceph-mon[95978]: pgmap v314: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:21.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:20 smithi116 ceph-mon[85502]: pgmap v314: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:22.035 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:08:22.035 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:08:23.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:22 smithi116 ceph-mon[85502]: pgmap v315: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:23.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:22 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/3650938071' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:23.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:22 smithi012 ceph-mon[91852]: pgmap v315: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:23.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:22 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/3650938071' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:23.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:22 smithi012 ceph-mon[95978]: pgmap v315: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:23.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:22 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/3650938071' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:25.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:24 smithi116 ceph-mon[85502]: pgmap v316: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:25.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:24 smithi012 ceph-mon[91852]: pgmap v316: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:25.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:24 smithi012 ceph-mon[95978]: pgmap v316: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:25.637 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:08:25.914 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:08:26.947 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:26 smithi012 ceph-mon[91852]: pgmap v317: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:26.947 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:26 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:08:26.947 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:26 smithi012 ceph-mon[95978]: pgmap v317: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:26.947 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:26 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:08:27.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:26 smithi116 ceph-mon[85502]: pgmap v317: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:27.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:26 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:08:27.706 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:08:27.706 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:08:28.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:27 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/1114341796' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:28.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:27 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/1114341796' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:28.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:27 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/1114341796' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:28.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:08:27 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:08:27] "GET /metrics HTTP/1.1" 200 34018 "" "Prometheus/2.43.0" 2024-04-26T05:08:29.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:28 smithi116 ceph-mon[85502]: pgmap v318: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:29.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:28 smithi012 ceph-mon[91852]: pgmap v318: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:29.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:28 smithi012 ceph-mon[95978]: pgmap v318: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:31.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:30 smithi116 ceph-mon[85502]: pgmap v319: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:31.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:30 smithi012 ceph-mon[91852]: pgmap v319: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:31.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:30 smithi012 ceph-mon[95978]: pgmap v319: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:31.561 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:08:31.838 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:08:33.147 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:32 smithi012 ceph-mon[91852]: pgmap v320: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:33.147 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:32 smithi012 ceph-mon[95978]: pgmap v320: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:33.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:32 smithi116 ceph-mon[85502]: pgmap v320: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:33.814 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:08:33.814 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:08:34.143 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:33 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/1076449418' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:34.176 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:33 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/1076449418' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:34.176 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:33 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/1076449418' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:35.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:34 smithi116 ceph-mon[85502]: pgmap v321: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:35.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:34 smithi012 ceph-mon[91852]: pgmap v321: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:35.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:34 smithi012 ceph-mon[95978]: pgmap v321: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:37.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:36 smithi116 ceph-mon[85502]: pgmap v322: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:37.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:36 smithi012 ceph-mon[91852]: pgmap v322: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:37.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:36 smithi012 ceph-mon[95978]: pgmap v322: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:37.549 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:08:37.822 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:08:38.231 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:08:37 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:08:37] "GET /metrics HTTP/1.1" 200 34018 "" "Prometheus/2.43.0" 2024-04-26T05:08:39.155 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:38 smithi012 ceph-mon[91852]: pgmap v323: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:39.156 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:38 smithi012 ceph-mon[95978]: pgmap v323: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:39.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:38 smithi116 ceph-mon[85502]: pgmap v323: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:39.644 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:08:39.644 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:08:40.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:39 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/34963442' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:40.262 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:39 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/34963442' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:40.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:39 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/34963442' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:41.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:40 smithi116 ceph-mon[85502]: pgmap v324: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:41.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:40 smithi012 ceph-mon[91852]: pgmap v324: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:41.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:40 smithi012 ceph-mon[95978]: pgmap v324: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:42.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:41 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:08:42.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:41 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:08:42.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:41 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:08:43.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:42 smithi116 ceph-mon[85502]: pgmap v325: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:43.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:42 smithi012 ceph-mon[91852]: pgmap v325: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:43.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:42 smithi012 ceph-mon[95978]: pgmap v325: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:43.320 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:08:43.597 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:08:45.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:44 smithi012 ceph-mon[91852]: pgmap v326: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:45.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:44 smithi012 ceph-mon[95978]: pgmap v326: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:45.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:44 smithi116 ceph-mon[85502]: pgmap v326: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:45.534 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:08:45.535 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:08:46.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:45 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/2367047957' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:46.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:45 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/2367047957' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:46.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:45 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/2367047957' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:47.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:46 smithi116 ceph-mon[85502]: pgmap v327: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:47.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:46 smithi012 ceph-mon[91852]: pgmap v327: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:47.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:46 smithi012 ceph-mon[95978]: pgmap v327: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:48.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:08:47 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:08:47] "GET /metrics HTTP/1.1" 200 34018 "" "Prometheus/2.43.0" 2024-04-26T05:08:49.179 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:48 smithi012 ceph-mon[95978]: pgmap v328: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:49.179 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:48 smithi012 ceph-mon[91852]: pgmap v328: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:49.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:48 smithi116 ceph-mon[85502]: pgmap v328: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:49.315 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:08:49.591 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:08:51.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:50 smithi012 ceph-mon[91852]: pgmap v329: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:51.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:50 smithi012 ceph-mon[95978]: pgmap v329: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:51.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:50 smithi116 ceph-mon[85502]: pgmap v329: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:51.511 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:08:51.511 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:08:52.066 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:51 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/1450662116' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:52.066 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:51 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/1450662116' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:52.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:51 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/1450662116' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:53.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:52 smithi116 ceph-mon[85502]: pgmap v330: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:53.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:52 smithi012 ceph-mon[91852]: pgmap v330: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:53.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:52 smithi012 ceph-mon[95978]: pgmap v330: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:55.093 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:08:55.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:54 smithi116 ceph-mon[85502]: pgmap v331: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:55.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:54 smithi012 ceph-mon[91852]: pgmap v331: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:55.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:54 smithi012 ceph-mon[95978]: pgmap v331: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:55.373 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:08:57.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:56 smithi012 ceph-mon[95978]: pgmap v332: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:57.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:56 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:08:57.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:56 smithi012 ceph-mon[91852]: pgmap v332: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:57.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:56 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:08:57.240 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:08:57.240 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:08:57.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:56 smithi116 ceph-mon[85502]: pgmap v332: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:57.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:56 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:08:58.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:57 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/672026099' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:58.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:57 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/672026099' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:58.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:57 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/672026099' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:08:58.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:08:57 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:08:57] "GET /metrics HTTP/1.1" 200 34019 "" "Prometheus/2.43.0" 2024-04-26T05:08:59.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:08:58 smithi116 ceph-mon[85502]: pgmap v333: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:59.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:08:58 smithi012 ceph-mon[91852]: pgmap v333: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:08:59.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:08:58 smithi012 ceph-mon[95978]: pgmap v333: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:00.958 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:09:01.234 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:09:01.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:00 smithi116 ceph-mon[85502]: pgmap v334: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:01.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:00 smithi012 ceph-mon[91852]: pgmap v334: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:01.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:00 smithi012 ceph-mon[95978]: pgmap v334: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:03.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:02 smithi012 ceph-mon[91852]: pgmap v335: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:03.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:02 smithi012 ceph-mon[95978]: pgmap v335: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:03.176 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:09:03.177 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:09:03.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:02 smithi116 ceph-mon[85502]: pgmap v335: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:04.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:03 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/1159264422' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:04.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:03 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/1159264422' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:04.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:03 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/1159264422' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:05.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:04 smithi116 ceph-mon[85502]: pgmap v336: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:05.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:04 smithi012 ceph-mon[91852]: pgmap v336: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:05.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:04 smithi012 ceph-mon[95978]: pgmap v336: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:06.863 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:09:07.141 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:09:07.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:06 smithi116 ceph-mon[85502]: pgmap v337: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:07.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:06 smithi012 ceph-mon[91852]: pgmap v337: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:07.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:06 smithi012 ceph-mon[95978]: pgmap v337: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:08.232 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:09:07 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:09:07] "GET /metrics HTTP/1.1" 200 34019 "" "Prometheus/2.43.0" 2024-04-26T05:09:09.008 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:09:09.008 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:09:09.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:08 smithi116 ceph-mon[85502]: pgmap v338: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:09.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:08 smithi012 ceph-mon[95978]: pgmap v338: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:09.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:08 smithi012 ceph-mon[91852]: pgmap v338: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:10.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:09 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/3927917215' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:10.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:09 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:09:10.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:09 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/3927917215' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:10.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:09 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:09:10.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:09 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/3927917215' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:10.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:09 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:09:11.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:10 smithi012 ceph-mon[95978]: pgmap v339: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:11.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:10 smithi012 ceph-mon[91852]: pgmap v339: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:11.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:10 smithi116 ceph-mon[85502]: pgmap v339: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:12.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:11 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:09:12.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:11 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:09:12.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:11 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:09:12.610 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:09:12.984 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:09:13.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:12 smithi116 ceph-mon[85502]: pgmap v340: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:13.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:12 smithi012 ceph-mon[91852]: pgmap v340: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:13.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:12 smithi012 ceph-mon[95978]: pgmap v340: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:14.922 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:14 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:09:14.923 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:14 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:09:14.923 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:14 smithi012 ceph-mon[91852]: pgmap v341: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:14.923 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:14 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:09:14.923 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:14 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:09:14.923 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:14 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:09:14.923 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:14 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:09:14.923 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:14 smithi012 ceph-mon[95978]: pgmap v341: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:14.923 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:14 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:09:14.923 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:14 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:09:14.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:14 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:09:14.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:14 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:09:14.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:14 smithi116 ceph-mon[85502]: pgmap v341: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:14.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:14 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:09:14.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:14 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:09:15.711 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:09:15.711 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:09:15.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:15 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:09:15.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:15 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:09:15.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:15 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:09:16.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:15 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:09:16.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:15 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:09:16.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:15 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:09:16.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:15 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:09:16.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:15 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:09:16.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:15 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:09:16.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:16 smithi116 ceph-mon[85502]: pgmap v342: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:16.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:16 smithi116 ceph-mon[85502]: pgmap v343: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:16.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:16 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/1816680278' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:17.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:16 smithi012 ceph-mon[91852]: pgmap v342: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:17.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:16 smithi012 ceph-mon[91852]: pgmap v343: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:17.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:16 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/1816680278' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:17.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:16 smithi012 ceph-mon[95978]: pgmap v342: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:17.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:16 smithi012 ceph-mon[95978]: pgmap v343: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:17.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:16 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/1816680278' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:18.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:09:17 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:09:17] "GET /metrics HTTP/1.1" 200 34014 "" "Prometheus/2.43.0" 2024-04-26T05:09:18.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:18 smithi116 ceph-mon[85502]: pgmap v344: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:19.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:18 smithi012 ceph-mon[91852]: pgmap v344: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:19.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:18 smithi012 ceph-mon[95978]: pgmap v344: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:19.325 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:09:19.600 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:09:20.884 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:20 smithi012 ceph-mon[91852]: pgmap v345: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:20.885 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:20 smithi012 ceph-mon[95978]: pgmap v345: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:20.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:20 smithi116 ceph-mon[85502]: pgmap v345: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:21.364 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:09:21.364 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:09:21.970 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:21 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/2593193098' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:21.970 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:21 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/2593193098' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:21.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:21 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/2593193098' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:22.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:22 smithi116 ceph-mon[85502]: pgmap v346: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:23.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:22 smithi012 ceph-mon[91852]: pgmap v346: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:23.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:22 smithi012 ceph-mon[95978]: pgmap v346: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:24.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:24 smithi116 ceph-mon[85502]: pgmap v347: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:24.998 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:09:25.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:24 smithi012 ceph-mon[91852]: pgmap v347: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:25.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:24 smithi012 ceph-mon[95978]: pgmap v347: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:25.274 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:09:26.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:26 smithi116 ceph-mon[85502]: pgmap v348: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:26.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:26 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:09:27.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:26 smithi012 ceph-mon[91852]: pgmap v348: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:27.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:26 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:09:27.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:26 smithi012 ceph-mon[95978]: pgmap v348: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:27.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:26 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:09:27.127 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:09:27.128 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:09:27.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:27 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/618368835' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:27.992 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:27 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/618368835' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:27.992 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:27 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/618368835' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:28.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:09:27 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:09:27] "GET /metrics HTTP/1.1" 200 34019 "" "Prometheus/2.43.0" 2024-04-26T05:09:28.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:28 smithi116 ceph-mon[85502]: pgmap v349: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:29.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:28 smithi012 ceph-mon[91852]: pgmap v349: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:29.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:28 smithi012 ceph-mon[95978]: pgmap v349: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:30.698 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:09:30.972 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:09:30.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:30 smithi116 ceph-mon[85502]: pgmap v350: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:31.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:30 smithi012 ceph-mon[91852]: pgmap v350: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:31.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:30 smithi012 ceph-mon[95978]: pgmap v350: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:32.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:32 smithi012 ceph-mon[91852]: pgmap v351: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:32.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:32 smithi012 ceph-mon[95978]: pgmap v351: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:32.960 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:09:32.960 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:09:32.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:32 smithi116 ceph-mon[85502]: pgmap v351: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:33.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:33 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/2054852917' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:34.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:33 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/2054852917' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:34.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:33 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/2054852917' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:34.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:34 smithi116 ceph-mon[85502]: pgmap v352: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:35.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:34 smithi012 ceph-mon[91852]: pgmap v352: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:35.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:34 smithi012 ceph-mon[95978]: pgmap v352: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:36.628 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:09:36.906 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:09:36.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:36 smithi116 ceph-mon[85502]: pgmap v353: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:37.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:36 smithi012 ceph-mon[91852]: pgmap v353: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:37.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:36 smithi012 ceph-mon[95978]: pgmap v353: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:38.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:09:37 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:09:37] "GET /metrics HTTP/1.1" 200 34019 "" "Prometheus/2.43.0" 2024-04-26T05:09:38.833 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:09:38.833 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:09:38.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:38 smithi116 ceph-mon[85502]: pgmap v354: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:39.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:38 smithi012 ceph-mon[95978]: pgmap v354: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:39.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:38 smithi012 ceph-mon[91852]: pgmap v354: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:39.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:39 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/1803920344' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:39.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:39 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/1803920344' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:39.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:39 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/1803920344' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:40.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:40 smithi116 ceph-mon[85502]: pgmap v355: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:41.012 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:40 smithi012 ceph-mon[95978]: pgmap v355: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:41.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:40 smithi012 ceph-mon[91852]: pgmap v355: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:41.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:41 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:09:42.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:41 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:09:42.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:41 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:09:42.537 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:09:42.818 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:09:42.940 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:42 smithi012 ceph-mon[91852]: pgmap v356: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:42.941 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:42 smithi012 ceph-mon[95978]: pgmap v356: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:42.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:42 smithi116 ceph-mon[85502]: pgmap v356: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:44.607 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:09:44.608 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:09:44.947 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:44 smithi012 ceph-mon[91852]: pgmap v357: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:44.947 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:44 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/1123526784' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:44.947 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:44 smithi012 ceph-mon[95978]: pgmap v357: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:44.947 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:44 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/1123526784' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:44.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:44 smithi116 ceph-mon[85502]: pgmap v357: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:44.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:44 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/1123526784' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:46.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:46 smithi116 ceph-mon[85502]: pgmap v358: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:47.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:46 smithi012 ceph-mon[95978]: pgmap v358: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:47.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:46 smithi012 ceph-mon[91852]: pgmap v358: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:48.217 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:09:48.245 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:09:47 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:09:47] "GET /metrics HTTP/1.1" 200 34019 "" "Prometheus/2.43.0" 2024-04-26T05:09:48.501 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:09:48.796 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:48 smithi012 ceph-mon[91852]: pgmap v359: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:48.797 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:48 smithi012 ceph-mon[95978]: pgmap v359: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:48.994 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:48 smithi116 ceph-mon[85502]: pgmap v359: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:50.416 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:09:50.416 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:09:50.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:50 smithi116 ceph-mon[85502]: pgmap v360: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:50.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:50 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/772352102' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:51.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:50 smithi012 ceph-mon[91852]: pgmap v360: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:51.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:50 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/772352102' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:51.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:50 smithi012 ceph-mon[95978]: pgmap v360: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:51.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:50 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/772352102' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:52.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:52 smithi116 ceph-mon[85502]: pgmap v361: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:53.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:52 smithi012 ceph-mon[91852]: pgmap v361: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:53.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:52 smithi012 ceph-mon[95978]: pgmap v361: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:54.256 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:09:54.537 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:09:54.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:54 smithi116 ceph-mon[85502]: pgmap v362: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:54.994 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:54 smithi012 ceph-mon[91852]: pgmap v362: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:54.994 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:54 smithi012 ceph-mon[95978]: pgmap v362: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:56.645 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:09:56.645 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:09:56.994 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:56 smithi116 ceph-mon[85502]: pgmap v363: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:56.994 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:56 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:09:56.994 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:56 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/71676108' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:57.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:56 smithi012 ceph-mon[91852]: pgmap v363: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:57.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:56 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:09:57.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:56 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/71676108' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:57.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:56 smithi012 ceph-mon[95978]: pgmap v363: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:57.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:56 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:09:57.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:56 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/71676108' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:09:58.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:09:57 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:09:57] "GET /metrics HTTP/1.1" 200 34018 "" "Prometheus/2.43.0" 2024-04-26T05:09:58.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:09:58 smithi116 ceph-mon[85502]: pgmap v364: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:59.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:09:58 smithi012 ceph-mon[91852]: pgmap v364: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:09:59.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:09:58 smithi012 ceph-mon[95978]: pgmap v364: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:00.360 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:10:00.642 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:10:01.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:00 smithi012 ceph-mon[91852]: pgmap v365: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:01.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:00 smithi012 ceph-mon[91852]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T05:10:01.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:00 smithi012 ceph-mon[95978]: pgmap v365: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:01.014 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:00 smithi012 ceph-mon[95978]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T05:10:01.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:00 smithi116 ceph-mon[85502]: pgmap v365: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:01.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:00 smithi116 ceph-mon[85502]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T05:10:02.584 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:10:02.585 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:10:02.872 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:02 smithi012 ceph-mon[91852]: pgmap v366: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:02.872 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:02 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/1769740301' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:02.873 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:02 smithi012 ceph-mon[95978]: pgmap v366: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:02.873 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:02 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/1769740301' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:03.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:02 smithi116 ceph-mon[85502]: pgmap v366: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:03.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:02 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/1769740301' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:05.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:04 smithi012 ceph-mon[91852]: pgmap v367: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:05.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:04 smithi012 ceph-mon[95978]: pgmap v367: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:05.042 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:04 smithi116 ceph-mon[85502]: pgmap v367: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:06.260 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:10:06.538 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:10:06.970 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:06 smithi012 ceph-mon[91852]: pgmap v368: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:06.970 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:06 smithi012 ceph-mon[95978]: pgmap v368: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:07.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:06 smithi116 ceph-mon[85502]: pgmap v368: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:08.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:10:07 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:10:07] "GET /metrics HTTP/1.1" 200 34018 "" "Prometheus/2.43.0" 2024-04-26T05:10:08.431 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:10:08.431 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:10:09.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:08 smithi012 ceph-mon[95978]: pgmap v369: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:09.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:08 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/259473788' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:09.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:08 smithi012 ceph-mon[91852]: pgmap v369: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:09.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:08 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/259473788' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:09.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:08 smithi116 ceph-mon[85502]: pgmap v369: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:09.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:08 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/259473788' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:11.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:10 smithi012 ceph-mon[91852]: pgmap v370: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:11.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:10 smithi012 ceph-mon[95978]: pgmap v370: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:11.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:10 smithi116 ceph-mon[85502]: pgmap v370: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:12.193 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:10:12.221 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:11 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:10:12.222 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:11 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:10:12.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:11 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:10:12.469 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:10:12.901 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:12 smithi012 ceph-mon[91852]: pgmap v371: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:12.901 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:12 smithi012 ceph-mon[95978]: pgmap v371: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:13.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:12 smithi116 ceph-mon[85502]: pgmap v371: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:14.381 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:10:14.381 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:10:14.999 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:14 smithi012 ceph-mon[95978]: pgmap v372: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:14.999 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:14 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/3340772151' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:14.999 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:14 smithi012 ceph-mon[91852]: pgmap v372: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:14.999 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:14 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/3340772151' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:15.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:14 smithi116 ceph-mon[85502]: pgmap v372: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:15.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:14 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/3340772151' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:16.156 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:15 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:10:16.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:15 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:10:16.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:15 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:10:16.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:16 smithi116 ceph-mon[85502]: pgmap v373: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:16.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:16 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:10:16.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:16 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:10:17.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:16 smithi012 ceph-mon[95978]: pgmap v373: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:17.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:16 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:10:17.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:16 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:10:17.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:16 smithi012 ceph-mon[91852]: pgmap v373: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:17.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:16 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:10:17.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:16 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:10:18.030 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:10:18.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:10:17 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:10:17] "GET /metrics HTTP/1.1" 200 34017 "" "Prometheus/2.43.0" 2024-04-26T05:10:18.875 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:10:19.099 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:18 smithi012 ceph-mon[91852]: pgmap v374: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:19.100 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:18 smithi012 ceph-mon[95978]: pgmap v374: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:19.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:18 smithi116 ceph-mon[85502]: pgmap v374: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:20.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:20 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:10:20.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:20 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:10:20.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:20 smithi012 ceph-mon[95978]: pgmap v375: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:20.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:20 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:10:20.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:20 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:10:20.264 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:20 smithi012 ceph-mon[91852]: pgmap v375: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:20.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:20 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:10:20.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:20 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:10:20.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:20 smithi116 ceph-mon[85502]: pgmap v375: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:21.507 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:10:21.508 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:10:22.097 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:21 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:10:22.097 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:21 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:10:22.097 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:21 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/3886871728' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:22.097 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:21 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:10:22.097 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:21 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:10:22.097 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:21 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:10:22.097 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:21 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:10:22.098 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:21 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:10:22.098 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:21 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/3886871728' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:22.098 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:21 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:10:22.098 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:21 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:10:22.098 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:21 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:10:22.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:21 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:10:22.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:21 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:10:22.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:21 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/3886871728' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:22.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:21 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:10:22.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:21 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:10:22.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:21 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' 2024-04-26T05:10:23.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:22 smithi116 ceph-mon[85502]: pgmap v376: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:23.241 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:22 smithi116 ceph-mon[85502]: pgmap v377: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:23.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:22 smithi012 ceph-mon[91852]: pgmap v376: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:23.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:22 smithi012 ceph-mon[91852]: pgmap v377: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:23.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:22 smithi012 ceph-mon[95978]: pgmap v376: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:23.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:22 smithi012 ceph-mon[95978]: pgmap v377: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:25.123 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:10:25.242 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:24 smithi116 ceph-mon[85502]: pgmap v378: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:25.262 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:24 smithi012 ceph-mon[91852]: pgmap v378: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:25.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:24 smithi012 ceph-mon[95978]: pgmap v378: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:25.402 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:10:27.125 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:26 smithi012 ceph-mon[91852]: pgmap v379: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:27.125 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:26 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:10:27.125 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:26 smithi012 ceph-mon[95978]: pgmap v379: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:27.125 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:26 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:10:27.366 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:10:27.367 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:10:27.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:26 smithi116 ceph-mon[85502]: pgmap v379: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:27.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:26 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:10:28.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:27 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/2689687193' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:28.013 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:10:27 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:10:27] "GET /metrics HTTP/1.1" 200 34016 "" "Prometheus/2.43.0" 2024-04-26T05:10:28.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:27 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/2689687193' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:28.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:27 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/2689687193' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:29.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:28 smithi012 ceph-mon[91852]: pgmap v380: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:29.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:29 smithi012 ceph-mon[95978]: pgmap v380: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:29.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:28 smithi116 ceph-mon[85502]: pgmap v380: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:31.039 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:10:31.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:31 smithi012 ceph-mon[91852]: pgmap v381: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:31.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:31 smithi012 ceph-mon[95978]: pgmap v381: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:31.317 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:10:31.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:31 smithi116 ceph-mon[85502]: pgmap v381: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:32.458 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:32 smithi012 ceph-mon[91852]: pgmap v382: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:32.459 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:32 smithi012 ceph-mon[95978]: pgmap v382: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:32.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:32 smithi116 ceph-mon[85502]: pgmap v382: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:33.356 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:10:33.356 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:10:33.646 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:33 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/2166494235' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:33.646 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:33 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/2166494235' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:33.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:33 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/2166494235' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:34.741 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:34 smithi116 ceph-mon[85502]: pgmap v383: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:34.762 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:34 smithi012 ceph-mon[91852]: pgmap v383: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:34.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:34 smithi012 ceph-mon[95978]: pgmap v383: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:36.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:36 smithi116 ceph-mon[85502]: pgmap v384: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:37.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:36 smithi012 ceph-mon[91852]: pgmap v384: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:37.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:36 smithi012 ceph-mon[95978]: pgmap v384: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:37.030 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:10:37.310 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:10:38.227 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:10:37 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:10:37] "GET /metrics HTTP/1.1" 200 34016 "" "Prometheus/2.43.0" 2024-04-26T05:10:38.859 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:38 smithi012 ceph-mon[91852]: pgmap v385: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:38.860 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:38 smithi012 ceph-mon[95978]: pgmap v385: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:38.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:38 smithi116 ceph-mon[85502]: pgmap v385: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:39.366 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:10:39.366 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:10:39.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:39 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/3454227197' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:40.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:39 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/3454227197' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:40.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:39 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/3454227197' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:40.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:40 smithi116 ceph-mon[85502]: pgmap v386: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:41.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:40 smithi012 ceph-mon[91852]: pgmap v386: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:41.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:40 smithi012 ceph-mon[95978]: pgmap v386: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:41.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:41 smithi116 ceph-mon[85502]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:10:42.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:41 smithi012 ceph-mon[91852]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:10:42.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:41 smithi012 ceph-mon[95978]: from='mgr.34107 172.21.15.12:0/496364431' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:10:42.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:42 smithi116 ceph-mon[85502]: pgmap v387: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:43.013 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:42 smithi012 ceph-mon[91852]: pgmap v387: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:43.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:42 smithi012 ceph-mon[95978]: pgmap v387: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:43.097 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:10:43.372 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:10:44.763 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:44 smithi012 ceph-mon[91852]: pgmap v388: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:44.763 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:44 smithi012 ceph-mon[95978]: pgmap v388: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:44.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:44 smithi116 ceph-mon[85502]: pgmap v388: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:45.278 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:10:45.278 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:10:45.971 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:45 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/3544715497' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:45.972 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:45 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/3544715497' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:45.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:45 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/3544715497' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:46.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:46 smithi116 ceph-mon[85502]: pgmap v389: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:47.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:46 smithi012 ceph-mon[91852]: pgmap v389: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:47.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:46 smithi012 ceph-mon[95978]: pgmap v389: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:48.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:10:47 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: ::ffff:172.21.15.116 - - [26/Apr/2024:05:10:47] "GET /metrics HTTP/1.1" 200 34010 "" "Prometheus/2.43.0" 2024-04-26T05:10:48.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:48 smithi116 ceph-mon[85502]: pgmap v390: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:49.003 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:10:49.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:48 smithi012 ceph-mon[91852]: pgmap v390: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:49.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:48 smithi012 ceph-mon[95978]: pgmap v390: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:49.268 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:10:50.991 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:50 smithi116 ceph-mon[85502]: pgmap v391: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:51.012 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:50 smithi012 ceph-mon[91852]: pgmap v391: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:51.013 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:50 smithi012 ceph-mon[95978]: pgmap v391: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-26T05:10:51.229 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-04-26T05:10:51.229 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:10:51.875 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:51 smithi012 ceph-mon[91852]: from='client.? 172.21.15.12:0/1352828199' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:51.875 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:51 smithi012 ceph-mon[95978]: from='client.? 172.21.15.12:0/1352828199' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:51.904 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa/tasks/ceph.py", line 1478, in healthy manager.wait_until_healthy(timeout=300) File "/home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa/tasks/ceph_manager.py", line 3204, in wait_until_healthy assert time.time() - start < timeout, \ AssertionError: timeout expired in wait_until_healthy 2024-04-26T05:10:52.156 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=6c5aaed31b5d4770a9b5b8c69e943148 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa/tasks/ceph.py", line 1478, in healthy manager.wait_until_healthy(timeout=300) File "/home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa/tasks/ceph_manager.py", line 3204, in wait_until_healthy assert time.time() - start < timeout, \ AssertionError: timeout expired in wait_until_healthy 2024-04-26T05:10:52.158 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-04-26T05:10:52.168 INFO:tasks.cephadm:Teardown begin 2024-04-26T05:10:52.168 DEBUG:teuthology.orchestra.run.smithi012:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-26T05:10:52.172 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:51 smithi116 ceph-mon[85502]: from='client.? 172.21.15.12:0/1352828199' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:10:52.204 DEBUG:teuthology.orchestra.run.smithi116:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-26T05:10:52.236 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-04-26T05:10:52.237 DEBUG:teuthology.orchestra.run.smithi012:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-26T05:10:52.264 DEBUG:teuthology.orchestra.run.smithi116:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-26T05:10:52.294 INFO:tasks.cephadm:Stopping all daemons... 2024-04-26T05:10:52.294 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-04-26T05:10:52.294 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl stop ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mon.a 2024-04-26T05:10:52.513 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:52 smithi012 systemd[1]: Stopping Ceph mon.a for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T05:10:52.963 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:52 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-a[91848]: 2024-04-26T05:10:52.619+0000 7f8b46f22700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:10:52.963 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:52 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-a[91848]: 2024-04-26T05:10:52.619+0000 7f8b46f22700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-04-26T05:10:52.964 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:52 smithi012 podman[137906]: 2024-04-26 05:10:52.681630355 +0000 UTC m=+0.185824790 container died db1f873383b097011eb4f61089abb64565ce058ed9342ec7badb8ba4c09eb414 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-a, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424) 2024-04-26T05:10:52.964 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:10:52 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: [26/Apr/2024:05:10:52] ENGINE Bus STOPPING 2024-04-26T05:10:52.964 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:10:52 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: [26/Apr/2024:05:10:52] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T05:10:52.964 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:10:52 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: [26/Apr/2024:05:10:52] ENGINE Bus STOPPED 2024-04-26T05:10:52.964 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:10:52 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: [26/Apr/2024:05:10:52] ENGINE Bus STARTING 2024-04-26T05:10:52.964 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:10:52 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: [26/Apr/2024:05:10:52] ENGINE Serving on http://:::9283 2024-04-26T05:10:52.965 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:10:52 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: [26/Apr/2024:05:10:52] ENGINE Bus STARTED 2024-04-26T05:10:53.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:53 smithi012 podman[137906]: 2024-04-26 05:10:53.062408724 +0000 UTC m=+0.566603149 container cleanup db1f873383b097011eb4f61089abb64565ce058ed9342ec7badb8ba4c09eb414 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-a, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T05:10:53.263 INFO:journalctl@ceph.mon.a.smithi012.stdout:Apr 26 05:10:53 smithi012 bash[137906]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-a 2024-04-26T05:10:53.539 DEBUG:teuthology.orchestra.run.smithi012:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mon.a.service' 2024-04-26T05:10:53.583 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:10:53.583 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-04-26T05:10:53.584 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-04-26T05:10:53.584 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl stop ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mon.c 2024-04-26T05:10:53.762 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:53 smithi012 systemd[1]: Stopping Ceph mon.c for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T05:10:54.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:53 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-c[95973]: 2024-04-26T05:10:53.899+0000 7f2886c9f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:10:54.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:53 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-c[95973]: 2024-04-26T05:10:53.899+0000 7f2886c9f700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-04-26T05:10:54.263 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:54 smithi012 podman[138040]: 2024-04-26 05:10:54.002367193 +0000 UTC m=+0.198374503 container died 4d75f0bd2f162ca53e9f6362694300674ca81993334806c816eb2ca3ea89a398 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-c, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-04-26T05:10:54.744 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:54 smithi012 podman[138040]: 2024-04-26 05:10:54.450570134 +0000 UTC m=+0.646577439 container cleanup 4d75f0bd2f162ca53e9f6362694300674ca81993334806c816eb2ca3ea89a398 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-c, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424) 2024-04-26T05:10:54.744 INFO:journalctl@ceph.mon.c.smithi012.stdout:Apr 26 05:10:54 smithi012 bash[138040]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-c 2024-04-26T05:10:54.893 DEBUG:teuthology.orchestra.run.smithi012:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mon.c.service' 2024-04-26T05:10:54.927 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:10:54.928 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-04-26T05:10:54.928 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-04-26T05:10:54.928 DEBUG:teuthology.orchestra.run.smithi116:> sudo systemctl stop ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mon.b 2024-04-26T05:10:55.224 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:54 smithi116 systemd[1]: Stopping Ceph mon.b for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T05:10:55.491 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:55 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-b[85498]: 2024-04-26T05:10:55.223+0000 7fa5bdee8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:10:55.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:55 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-b[85498]: 2024-04-26T05:10:55.223+0000 7fa5bdee8700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-04-26T05:10:55.492 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:55 smithi116 podman[95622]: 2024-04-26 05:10:55.244348635 +0000 UTC m=+0.124663040 container died a0c4cbd180dca3a01b2950b2898656630d35f42c2a80b12bb4179d5ece61178f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-b, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240424) 2024-04-26T05:10:55.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:55 smithi116 podman[95622]: 2024-04-26 05:10:55.70695088 +0000 UTC m=+0.587265328 container cleanup a0c4cbd180dca3a01b2950b2898656630d35f42c2a80b12bb4179d5ece61178f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-b, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T05:10:55.992 INFO:journalctl@ceph.mon.b.smithi116.stdout:Apr 26 05:10:55 smithi116 bash[95622]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mon-b 2024-04-26T05:10:56.173 DEBUG:teuthology.orchestra.run.smithi116:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mon.b.service' 2024-04-26T05:10:56.219 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:10:56.220 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-04-26T05:10:56.220 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-04-26T05:10:56.220 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl stop ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mgr.y 2024-04-26T05:10:56.513 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:10:56 smithi012 systemd[1]: Stopping Ceph mgr.y for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T05:10:56.849 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:10:56 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T05:10:56.529+0000 7f0e55a5f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:10:56.849 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:10:56 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y[82733]: 2024-04-26T05:10:56.529+0000 7f0e55a5f700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-04-26T05:10:56.849 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:10:56 smithi012 podman[138162]: 2024-04-26 05:10:56.561867036 +0000 UTC m=+0.152673318 container died 3a46d00317727c28523c6b1f8b84b372e3e8f8f1382fb10ef9d709d4703b0716 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-04-26T05:10:57.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:10:57 smithi012 podman[138162]: 2024-04-26 05:10:57.014600228 +0000 UTC m=+0.605406543 container cleanup 3a46d00317727c28523c6b1f8b84b372e3e8f8f1382fb10ef9d709d4703b0716 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y, GIT_CLEAN=True, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2) 2024-04-26T05:10:57.263 INFO:journalctl@ceph.mgr.y.smithi012.stdout:Apr 26 05:10:57 smithi012 bash[138162]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-y 2024-04-26T05:10:57.438 DEBUG:teuthology.orchestra.run.smithi012:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mgr.y.service' 2024-04-26T05:10:57.471 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:10:57.472 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-04-26T05:10:57.472 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-04-26T05:10:57.472 DEBUG:teuthology.orchestra.run.smithi116:> sudo systemctl stop ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mgr.x 2024-04-26T05:10:57.809 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 05:10:57 smithi116 systemd[1]: Stopping Ceph mgr.x for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T05:10:58.079 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 05:10:57 smithi116 podman[95743]: 2024-04-26 05:10:57.808684754 +0000 UTC m=+0.142817819 container died 5499bf989999b867b9e7b18933d59ab1e040e2fa379f78acdac5bc634799c3dc (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-04-26T05:10:58.438 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 05:10:58 smithi116 podman[95743]: 2024-04-26 05:10:58.194569764 +0000 UTC m=+0.528702846 container cleanup 5499bf989999b867b9e7b18933d59ab1e040e2fa379f78acdac5bc634799c3dc (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux ) 2024-04-26T05:10:58.438 INFO:journalctl@ceph.mgr.x.smithi116.stdout:Apr 26 05:10:58 smithi116 bash[95743]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968-mgr-x 2024-04-26T05:10:58.562 DEBUG:teuthology.orchestra.run.smithi116:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0b9350c6-0388-11ef-bc93-c7b262605968@mgr.x.service' 2024-04-26T05:10:58.595 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:10:58.595 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-04-26T05:10:58.595 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-04-26T05:10:58.596 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl stop ceph-0b9350c6-0388-11ef-bc93-c7b262605968@osd.0 2024-04-26T05:10:58.970 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:10:58 smithi012 systemd[1]: Stopping Ceph osd.0 for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T05:10:59.263 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:10:58 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0[103452]: 2024-04-26T05:10:58.968+0000 7fa58b6d7700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:10:59.263 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:10:58 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0[103452]: 2024-04-26T05:10:58.968+0000 7fa58b6d7700 -1 osd.0 100 *** Got signal Terminated *** 2024-04-26T05:10:59.263 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:10:58 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0[103452]: 2024-04-26T05:10:58.968+0000 7fa58b6d7700 -1 osd.0 100 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:11:04.754 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:11:04 smithi012 podman[138283]: 2024-04-26 05:11:04.378666928 +0000 UTC m=+5.590349266 container died 512ac46479c20e01647c0cc6f6ce4f1dd256baf85133d58dc3ec4437d19e9ad4 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux ) 2024-04-26T05:11:04.755 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 05:11:04 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[81716]: ts=2024-04-26T05:11:04.273Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi012.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.12:8443: connect: connection refused" 2024-04-26T05:11:04.755 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 05:11:04 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[81716]: ts=2024-04-26T05:11:04.273Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi116:8443/api/prometheus_receiver\": dial tcp 172.21.15.116:8443: connect: connection refused" 2024-04-26T05:11:05.013 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:11:04 smithi012 podman[138283]: 2024-04-26 05:11:04.893806101 +0000 UTC m=+6.105488448 container cleanup 512ac46479c20e01647c0cc6f6ce4f1dd256baf85133d58dc3ec4437d19e9ad4 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-04-26T05:11:05.014 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:11:04 smithi012 bash[138283]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0 2024-04-26T05:11:05.513 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:11:05 smithi012 podman[138300]: 2024-04-26 05:11:05.103610192 +0000 UTC m=+0.720647600 container remove 512ac46479c20e01647c0cc6f6ce4f1dd256baf85133d58dc3ec4437d19e9ad4 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, GIT_CLEAN=True) 2024-04-26T05:11:05.808 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:11:05 smithi012 podman[138387]: 2024-04-26 05:11:05.436143625 +0000 UTC m=+0.018692536 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:11:05.808 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:11:05 smithi012 podman[138387]: 2024-04-26 05:11:05.546463288 +0000 UTC m=+0.129012172 container create 6cda6716f8c45a1aef9aad1b393edc88a521f05fa4cfa7bd9c36ee529b193344 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0-deactivate, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T05:11:06.113 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:11:05 smithi012 podman[138387]: 2024-04-26 05:11:05.807593156 +0000 UTC m=+0.390142041 container init 6cda6716f8c45a1aef9aad1b393edc88a521f05fa4cfa7bd9c36ee529b193344 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0-deactivate, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5) 2024-04-26T05:11:06.113 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:11:05 smithi012 podman[138387]: 2024-04-26 05:11:05.811750478 +0000 UTC m=+0.394299363 container start 6cda6716f8c45a1aef9aad1b393edc88a521f05fa4cfa7bd9c36ee529b193344 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True) 2024-04-26T05:11:06.113 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:11:05 smithi012 podman[138387]: 2024-04-26 05:11:05.890686613 +0000 UTC m=+0.473235506 container attach 6cda6716f8c45a1aef9aad1b393edc88a521f05fa4cfa7bd9c36ee529b193344 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:11:06.393 INFO:journalctl@ceph.osd.0.smithi012.stdout:Apr 26 05:11:06 smithi012 podman[138387]: 2024-04-26 05:11:06.112728104 +0000 UTC m=+0.695277006 container died 6cda6716f8c45a1aef9aad1b393edc88a521f05fa4cfa7bd9c36ee529b193344 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-04-26T05:11:06.888 DEBUG:teuthology.orchestra.run.smithi012:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0b9350c6-0388-11ef-bc93-c7b262605968@osd.0.service' 2024-04-26T05:11:06.926 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:11:06.927 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-04-26T05:11:06.927 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-04-26T05:11:06.927 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl stop ceph-0b9350c6-0388-11ef-bc93-c7b262605968@osd.1 2024-04-26T05:11:07.263 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:11:07 smithi012 systemd[1]: Stopping Ceph osd.1 for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T05:11:07.762 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:11:07 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1[108954]: 2024-04-26T05:11:07.265+0000 7f8c6d050700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:11:07.763 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:11:07 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1[108954]: 2024-04-26T05:11:07.265+0000 7f8c6d050700 -1 osd.1 100 *** Got signal Terminated *** 2024-04-26T05:11:07.763 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:11:07 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1[108954]: 2024-04-26T05:11:07.265+0000 7f8c6d050700 -1 osd.1 100 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:11:12.875 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:11:12 smithi012 podman[138507]: 2024-04-26 05:11:12.612169476 +0000 UTC m=+5.461753297 container died 793612ebfa6621498967a884bf8911571e01d2817efd5da4928e13329b033ade (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-04-26T05:11:13.263 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:11:13 smithi012 podman[138507]: 2024-04-26 05:11:13.02368475 +0000 UTC m=+5.873268614 container cleanup 793612ebfa6621498967a884bf8911571e01d2817efd5da4928e13329b033ade (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-04-26T05:11:13.263 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:11:13 smithi012 bash[138507]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1 2024-04-26T05:11:13.763 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:11:13 smithi012 podman[138525]: 2024-04-26 05:11:13.342302127 +0000 UTC m=+0.726267808 container remove 793612ebfa6621498967a884bf8911571e01d2817efd5da4928e13329b033ade (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD) 2024-04-26T05:11:14.104 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:11:13 smithi012 podman[138618]: 2024-04-26 05:11:13.809811382 +0000 UTC m=+0.112119884 container create 75944371abdc242cc84cf736cf1856cc96f35c218f1fd2f1ef80015092d19320 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424) 2024-04-26T05:11:14.105 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:11:13 smithi012 podman[138618]: 2024-04-26 05:11:13.716649931 +0000 UTC m=+0.018958444 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:11:14.105 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:11:14 smithi012 podman[138618]: 2024-04-26 05:11:14.021172439 +0000 UTC m=+0.323480951 container init 75944371abdc242cc84cf736cf1856cc96f35c218f1fd2f1ef80015092d19320 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-04-26T05:11:14.105 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:11:14 smithi012 podman[138618]: 2024-04-26 05:11:14.024699 +0000 UTC m=+0.327007507 container start 75944371abdc242cc84cf736cf1856cc96f35c218f1fd2f1ef80015092d19320 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2) 2024-04-26T05:11:14.105 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:11:14 smithi012 podman[138618]: 2024-04-26 05:11:14.10396554 +0000 UTC m=+0.406274066 container attach 75944371abdc242cc84cf736cf1856cc96f35c218f1fd2f1ef80015092d19320 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1-deactivate, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True) 2024-04-26T05:11:14.513 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:11:14 smithi012 podman[138618]: 2024-04-26 05:11:14.331457853 +0000 UTC m=+0.633766357 container died 75944371abdc242cc84cf736cf1856cc96f35c218f1fd2f1ef80015092d19320 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1-deactivate, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True) 2024-04-26T05:11:14.513 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 05:11:14 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[81716]: ts=2024-04-26T05:11:14.273Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi012.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.12:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi116:8443/api/prometheus_receiver\": dial tcp 172.21.15.116:8443: connect: connection refused" 2024-04-26T05:11:14.513 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 05:11:14 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[81716]: ts=2024-04-26T05:11:14.274Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi012.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.12:8443: connect: connection refused" 2024-04-26T05:11:14.514 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 05:11:14 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[81716]: ts=2024-04-26T05:11:14.274Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi116:8443/api/prometheus_receiver\": dial tcp 172.21.15.116:8443: connect: connection refused" 2024-04-26T05:11:14.871 INFO:journalctl@ceph.osd.1.smithi012.stdout:Apr 26 05:11:14 smithi012 podman[138644]: 2024-04-26 05:11:14.857859736 +0000 UTC m=+0.521164058 container remove 75944371abdc242cc84cf736cf1856cc96f35c218f1fd2f1ef80015092d19320 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:11:14.877 DEBUG:teuthology.orchestra.run.smithi012:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0b9350c6-0388-11ef-bc93-c7b262605968@osd.1.service' 2024-04-26T05:11:14.916 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:11:14.916 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-04-26T05:11:14.916 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-04-26T05:11:14.916 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl stop ceph-0b9350c6-0388-11ef-bc93-c7b262605968@osd.2 2024-04-26T05:11:15.245 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:11:15 smithi012 systemd[1]: Stopping Ceph osd.2 for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T05:11:15.512 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:11:15 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2[113683]: 2024-04-26T05:11:15.243+0000 7f6e6e6db700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:11:15.513 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:11:15 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2[113683]: 2024-04-26T05:11:15.243+0000 7f6e6e6db700 -1 osd.2 100 *** Got signal Terminated *** 2024-04-26T05:11:15.513 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:11:15 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2[113683]: 2024-04-26T05:11:15.243+0000 7f6e6e6db700 -1 osd.2 100 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:11:20.978 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:11:20 smithi012 podman[138723]: 2024-04-26 05:11:20.682713907 +0000 UTC m=+5.535713933 container died 388804ce91742495d116e8ff604998919e22fe2247239619ae64e2e258fe597d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:11:21.263 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:11:21 smithi012 podman[138723]: 2024-04-26 05:11:21.159984489 +0000 UTC m=+6.012984513 container cleanup 388804ce91742495d116e8ff604998919e22fe2247239619ae64e2e258fe597d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-04-26T05:11:21.263 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:11:21 smithi012 bash[138723]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2 2024-04-26T05:11:21.763 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:11:21 smithi012 podman[138742]: 2024-04-26 05:11:21.461651257 +0000 UTC m=+0.769770050 container remove 388804ce91742495d116e8ff604998919e22fe2247239619ae64e2e258fe597d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:11:22.099 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:11:21 smithi012 podman[138836]: 2024-04-26 05:11:21.702303882 +0000 UTC m=+0.018434209 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:11:22.099 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:11:21 smithi012 podman[138836]: 2024-04-26 05:11:21.82942795 +0000 UTC m=+0.145558263 container create 7881de9010a2194db64e7ae4da66b6a28cbcdeef92df4584c40609b880da7c6f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb) 2024-04-26T05:11:22.100 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:11:22 smithi012 podman[138836]: 2024-04-26 05:11:22.032174263 +0000 UTC m=+0.348304575 container init 7881de9010a2194db64e7ae4da66b6a28cbcdeef92df4584c40609b880da7c6f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2-deactivate, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux ) 2024-04-26T05:11:22.100 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:11:22 smithi012 podman[138836]: 2024-04-26 05:11:22.035250426 +0000 UTC m=+0.351380749 container start 7881de9010a2194db64e7ae4da66b6a28cbcdeef92df4584c40609b880da7c6f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, GIT_CLEAN=True, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5) 2024-04-26T05:11:22.100 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:11:22 smithi012 podman[138836]: 2024-04-26 05:11:22.098786111 +0000 UTC m=+0.414916425 container attach 7881de9010a2194db64e7ae4da66b6a28cbcdeef92df4584c40609b880da7c6f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2-deactivate, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T05:11:22.362 INFO:journalctl@ceph.osd.2.smithi012.stdout:Apr 26 05:11:22 smithi012 podman[138836]: 2024-04-26 05:11:22.340917443 +0000 UTC m=+0.657047762 container died 7881de9010a2194db64e7ae4da66b6a28cbcdeef92df4584c40609b880da7c6f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:11:22.989 DEBUG:teuthology.orchestra.run.smithi012:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0b9350c6-0388-11ef-bc93-c7b262605968@osd.2.service' 2024-04-26T05:11:23.028 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:11:23.028 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-04-26T05:11:23.028 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-04-26T05:11:23.028 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl stop ceph-0b9350c6-0388-11ef-bc93-c7b262605968@osd.3 2024-04-26T05:11:23.262 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:11:23 smithi012 systemd[1]: Stopping Ceph osd.3 for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T05:11:23.612 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:11:23 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3[118582]: 2024-04-26T05:11:23.354+0000 7f7915c17700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:11:23.612 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:11:23 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3[118582]: 2024-04-26T05:11:23.355+0000 7f7915c17700 -1 osd.3 100 *** Got signal Terminated *** 2024-04-26T05:11:23.612 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:11:23 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3[118582]: 2024-04-26T05:11:23.355+0000 7f7915c17700 -1 osd.3 100 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:11:24.763 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 05:11:24 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[81716]: ts=2024-04-26T05:11:24.274Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi116:8443/api/prometheus_receiver\": dial tcp 172.21.15.116:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi012.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.12:8443: connect: connection refused" 2024-04-26T05:11:24.763 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 05:11:24 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[81716]: ts=2024-04-26T05:11:24.275Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi012.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.12:8443: connect: connection refused" 2024-04-26T05:11:24.763 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 05:11:24 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[81716]: ts=2024-04-26T05:11:24.275Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi116:8443/api/prometheus_receiver\": dial tcp 172.21.15.116:8443: connect: connection refused" 2024-04-26T05:11:25.741 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:25 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:25.283+0000 7fcc88b84700 -1 osd.5 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:00.724400+0000 front 2024-04-26T05:11:00.724363+0000 (oldest deadline 2024-04-26T05:11:24.824251+0000) 2024-04-26T05:11:26.491 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:26.238+0000 7fcc88b84700 -1 osd.5 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:00.724400+0000 front 2024-04-26T05:11:00.724363+0000 (oldest deadline 2024-04-26T05:11:24.824251+0000) 2024-04-26T05:11:26.492 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:26.127+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:01.789439+0000 front 2024-04-26T05:11:01.789480+0000 (oldest deadline 2024-04-26T05:11:25.289408+0000) 2024-04-26T05:11:26.492 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:26.010+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.316461+0000 front 2024-04-26T05:11:03.316324+0000 (oldest deadline 2024-04-26T05:11:25.616201+0000) 2024-04-26T05:11:26.991 INFO:journalctl@ceph.osd.4.smithi116.stdout:Apr 26 05:11:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-4[34852]: 2024-04-26T05:11:26.533+0000 7f9a53da3700 -1 osd.4 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.750291+0000 front 2024-04-26T05:11:03.750306+0000 (oldest deadline 2024-04-26T05:11:26.050247+0000) 2024-04-26T05:11:27.491 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:27 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:27.130+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:01.789439+0000 front 2024-04-26T05:11:01.789480+0000 (oldest deadline 2024-04-26T05:11:25.289408+0000) 2024-04-26T05:11:27.492 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:27 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:27.222+0000 7fcc88b84700 -1 osd.5 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:00.724400+0000 front 2024-04-26T05:11:00.724363+0000 (oldest deadline 2024-04-26T05:11:24.824251+0000) 2024-04-26T05:11:27.492 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:26 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:26.998+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.316461+0000 front 2024-04-26T05:11:03.316324+0000 (oldest deadline 2024-04-26T05:11:25.616201+0000) 2024-04-26T05:11:27.991 INFO:journalctl@ceph.osd.4.smithi116.stdout:Apr 26 05:11:27 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-4[34852]: 2024-04-26T05:11:27.538+0000 7f9a53da3700 -1 osd.4 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.750291+0000 front 2024-04-26T05:11:03.750306+0000 (oldest deadline 2024-04-26T05:11:26.050247+0000) 2024-04-26T05:11:28.493 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:28 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:28.185+0000 7fcc88b84700 -1 osd.5 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:00.724400+0000 front 2024-04-26T05:11:00.724363+0000 (oldest deadline 2024-04-26T05:11:24.824251+0000) 2024-04-26T05:11:28.493 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:28 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:28.131+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:01.789439+0000 front 2024-04-26T05:11:01.789480+0000 (oldest deadline 2024-04-26T05:11:25.289408+0000) 2024-04-26T05:11:28.494 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:28 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:28.016+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.316461+0000 front 2024-04-26T05:11:03.316324+0000 (oldest deadline 2024-04-26T05:11:25.616201+0000) 2024-04-26T05:11:28.922 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:11:28 smithi012 podman[138954]: 2024-04-26 05:11:28.670460165 +0000 UTC m=+5.404508929 container died a25aba82b703903a56ff4555b8df31635fa7f0f5b1e8efd592125fcb98e26e61 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-04-26T05:11:28.991 INFO:journalctl@ceph.osd.4.smithi116.stdout:Apr 26 05:11:28 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-4[34852]: 2024-04-26T05:11:28.493+0000 7f9a53da3700 -1 osd.4 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.750291+0000 front 2024-04-26T05:11:03.750306+0000 (oldest deadline 2024-04-26T05:11:26.050247+0000) 2024-04-26T05:11:29.257 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:11:29 smithi012 podman[138954]: 2024-04-26 05:11:29.113397538 +0000 UTC m=+5.847446343 container cleanup a25aba82b703903a56ff4555b8df31635fa7f0f5b1e8efd592125fcb98e26e61 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2) 2024-04-26T05:11:29.258 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:11:29 smithi012 bash[138954]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3 2024-04-26T05:11:29.492 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:29 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:29.180+0000 7fcc88b84700 -1 osd.5 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:00.724400+0000 front 2024-04-26T05:11:00.724363+0000 (oldest deadline 2024-04-26T05:11:24.824251+0000) 2024-04-26T05:11:29.492 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:29 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:29.106+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:01.789439+0000 front 2024-04-26T05:11:01.789480+0000 (oldest deadline 2024-04-26T05:11:25.289408+0000) 2024-04-26T05:11:29.493 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:29 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:29.050+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.316461+0000 front 2024-04-26T05:11:03.316324+0000 (oldest deadline 2024-04-26T05:11:25.616201+0000) 2024-04-26T05:11:29.512 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:11:29 smithi012 podman[138974]: 2024-04-26 05:11:29.256272158 +0000 UTC m=+0.581388642 container remove a25aba82b703903a56ff4555b8df31635fa7f0f5b1e8efd592125fcb98e26e61 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T05:11:29.985 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:11:29 smithi012 podman[139064]: 2024-04-26 05:11:29.592591694 +0000 UTC m=+0.018713001 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:11:29.985 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:11:29 smithi012 podman[139064]: 2024-04-26 05:11:29.690327739 +0000 UTC m=+0.116449034 container create b86861d7576c6b19043849b4b1a848c03b303fc6bb3dbf6048de6390dda8d612 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T05:11:29.985 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:11:29 smithi012 podman[139064]: 2024-04-26 05:11:29.90121426 +0000 UTC m=+0.327335558 container init b86861d7576c6b19043849b4b1a848c03b303fc6bb3dbf6048de6390dda8d612 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3-deactivate, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-04-26T05:11:29.985 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:11:29 smithi012 podman[139064]: 2024-04-26 05:11:29.904495977 +0000 UTC m=+0.330617273 container start b86861d7576c6b19043849b4b1a848c03b303fc6bb3dbf6048de6390dda8d612 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3-deactivate, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424) 2024-04-26T05:11:29.985 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:11:29 smithi012 podman[139064]: 2024-04-26 05:11:29.984411166 +0000 UTC m=+0.410532478 container attach b86861d7576c6b19043849b4b1a848c03b303fc6bb3dbf6048de6390dda8d612 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-04-26T05:11:29.991 INFO:journalctl@ceph.osd.4.smithi116.stdout:Apr 26 05:11:29 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-4[34852]: 2024-04-26T05:11:29.530+0000 7f9a53da3700 -1 osd.4 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.750291+0000 front 2024-04-26T05:11:03.750306+0000 (oldest deadline 2024-04-26T05:11:26.050247+0000) 2024-04-26T05:11:30.262 INFO:journalctl@ceph.osd.3.smithi012.stdout:Apr 26 05:11:30 smithi012 podman[139064]: 2024-04-26 05:11:30.208515035 +0000 UTC m=+0.634636331 container died b86861d7576c6b19043849b4b1a848c03b303fc6bb3dbf6048de6390dda8d612 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-04-26T05:11:30.491 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:30 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:30.147+0000 7fcc88b84700 -1 osd.5 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:00.724400+0000 front 2024-04-26T05:11:00.724363+0000 (oldest deadline 2024-04-26T05:11:24.824251+0000) 2024-04-26T05:11:30.492 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:30 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:30.139+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:01.789439+0000 front 2024-04-26T05:11:01.789480+0000 (oldest deadline 2024-04-26T05:11:25.289408+0000) 2024-04-26T05:11:30.492 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:30 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:30.086+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.316461+0000 front 2024-04-26T05:11:03.316324+0000 (oldest deadline 2024-04-26T05:11:25.616201+0000) 2024-04-26T05:11:30.829 DEBUG:teuthology.orchestra.run.smithi012:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0b9350c6-0388-11ef-bc93-c7b262605968@osd.3.service' 2024-04-26T05:11:30.870 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:11:30.871 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-04-26T05:11:30.871 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-04-26T05:11:30.871 DEBUG:teuthology.orchestra.run.smithi116:> sudo systemctl stop ceph-0b9350c6-0388-11ef-bc93-c7b262605968@osd.4 2024-04-26T05:11:30.899 INFO:journalctl@ceph.osd.4.smithi116.stdout:Apr 26 05:11:30 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-4[34852]: 2024-04-26T05:11:30.548+0000 7f9a53da3700 -1 osd.4 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.750291+0000 front 2024-04-26T05:11:03.750306+0000 (oldest deadline 2024-04-26T05:11:26.050247+0000) 2024-04-26T05:11:31.197 INFO:journalctl@ceph.osd.4.smithi116.stdout:Apr 26 05:11:30 smithi116 systemd[1]: Stopping Ceph osd.4 for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T05:11:31.197 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:31 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:31.140+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:01.789439+0000 front 2024-04-26T05:11:01.789480+0000 (oldest deadline 2024-04-26T05:11:25.289408+0000) 2024-04-26T05:11:31.197 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:31 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:31.140+0000 7fcc88b84700 -1 osd.5 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:00.724400+0000 front 2024-04-26T05:11:00.724363+0000 (oldest deadline 2024-04-26T05:11:24.824251+0000) 2024-04-26T05:11:31.198 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:31 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:31.125+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.316461+0000 front 2024-04-26T05:11:03.316324+0000 (oldest deadline 2024-04-26T05:11:25.616201+0000) 2024-04-26T05:11:31.491 INFO:journalctl@ceph.osd.4.smithi116.stdout:Apr 26 05:11:31 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-4[34852]: 2024-04-26T05:11:31.195+0000 7f9a567a3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:11:31.492 INFO:journalctl@ceph.osd.4.smithi116.stdout:Apr 26 05:11:31 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-4[34852]: 2024-04-26T05:11:31.195+0000 7f9a567a3700 -1 osd.4 100 *** Got signal Terminated *** 2024-04-26T05:11:31.492 INFO:journalctl@ceph.osd.4.smithi116.stdout:Apr 26 05:11:31 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-4[34852]: 2024-04-26T05:11:31.195+0000 7f9a567a3700 -1 osd.4 100 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:11:31.991 INFO:journalctl@ceph.osd.4.smithi116.stdout:Apr 26 05:11:31 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-4[34852]: 2024-04-26T05:11:31.580+0000 7f9a53da3700 -1 osd.4 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.750291+0000 front 2024-04-26T05:11:03.750306+0000 (oldest deadline 2024-04-26T05:11:26.050247+0000) 2024-04-26T05:11:32.491 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:32 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:32.134+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:01.789439+0000 front 2024-04-26T05:11:01.789480+0000 (oldest deadline 2024-04-26T05:11:25.289408+0000) 2024-04-26T05:11:32.492 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:32 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:32.187+0000 7fcc88b84700 -1 osd.5 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:00.724400+0000 front 2024-04-26T05:11:00.724363+0000 (oldest deadline 2024-04-26T05:11:24.824251+0000) 2024-04-26T05:11:32.492 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:32 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:32.119+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.316461+0000 front 2024-04-26T05:11:03.316324+0000 (oldest deadline 2024-04-26T05:11:25.616201+0000) 2024-04-26T05:11:32.991 INFO:journalctl@ceph.osd.4.smithi116.stdout:Apr 26 05:11:32 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-4[34852]: 2024-04-26T05:11:32.544+0000 7f9a53da3700 -1 osd.4 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.750291+0000 front 2024-04-26T05:11:03.750306+0000 (oldest deadline 2024-04-26T05:11:26.050247+0000) 2024-04-26T05:11:33.491 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:33 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:33.174+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:01.789439+0000 front 2024-04-26T05:11:01.789480+0000 (oldest deadline 2024-04-26T05:11:25.289408+0000) 2024-04-26T05:11:33.492 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:33 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:33.122+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.316461+0000 front 2024-04-26T05:11:03.316324+0000 (oldest deadline 2024-04-26T05:11:25.616201+0000) 2024-04-26T05:11:33.492 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:33 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:33.208+0000 7fcc88b84700 -1 osd.5 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:00.724400+0000 front 2024-04-26T05:11:00.724363+0000 (oldest deadline 2024-04-26T05:11:24.824251+0000) 2024-04-26T05:11:33.991 INFO:journalctl@ceph.osd.4.smithi116.stdout:Apr 26 05:11:33 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-4[34852]: 2024-04-26T05:11:33.575+0000 7f9a53da3700 -1 osd.4 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.750291+0000 front 2024-04-26T05:11:03.750306+0000 (oldest deadline 2024-04-26T05:11:26.050247+0000) 2024-04-26T05:11:34.491 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:34 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:34.159+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.316461+0000 front 2024-04-26T05:11:03.316324+0000 (oldest deadline 2024-04-26T05:11:25.616201+0000) 2024-04-26T05:11:34.492 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:34 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:34.200+0000 7fcc88b84700 -1 osd.5 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:00.724400+0000 front 2024-04-26T05:11:00.724363+0000 (oldest deadline 2024-04-26T05:11:24.824251+0000) 2024-04-26T05:11:34.492 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:34 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:34.200+0000 7fcc88b84700 -1 osd.5 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:11.225807+0000 front 2024-04-26T05:11:11.225760+0000 (oldest deadline 2024-04-26T05:11:33.525909+0000) 2024-04-26T05:11:34.493 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:34 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:34.156+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:01.789439+0000 front 2024-04-26T05:11:01.789480+0000 (oldest deadline 2024-04-26T05:11:25.289408+0000) 2024-04-26T05:11:34.493 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:34 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:34.156+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:09.390482+0000 front 2024-04-26T05:11:09.390548+0000 (oldest deadline 2024-04-26T05:11:34.090606+0000) 2024-04-26T05:11:34.763 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 05:11:34 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[81716]: ts=2024-04-26T05:11:34.274Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi116:8443/api/prometheus_receiver\": dial tcp 172.21.15.116:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi012.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.12:8443: connect: connection refused" 2024-04-26T05:11:34.763 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 05:11:34 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[81716]: ts=2024-04-26T05:11:34.276Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi116:8443/api/prometheus_receiver\": dial tcp 172.21.15.116:8443: connect: connection refused" 2024-04-26T05:11:34.763 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 05:11:34 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[81716]: ts=2024-04-26T05:11:34.276Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi012.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.12:8443: connect: connection refused" 2024-04-26T05:11:34.991 INFO:journalctl@ceph.osd.4.smithi116.stdout:Apr 26 05:11:34 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-4[34852]: 2024-04-26T05:11:34.580+0000 7f9a53da3700 -1 osd.4 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.750291+0000 front 2024-04-26T05:11:03.750306+0000 (oldest deadline 2024-04-26T05:11:26.050247+0000) 2024-04-26T05:11:35.491 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:35 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:35.112+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.316461+0000 front 2024-04-26T05:11:03.316324+0000 (oldest deadline 2024-04-26T05:11:25.616201+0000) 2024-04-26T05:11:35.492 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:35 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:35.199+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:01.789439+0000 front 2024-04-26T05:11:01.789480+0000 (oldest deadline 2024-04-26T05:11:25.289408+0000) 2024-04-26T05:11:35.492 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:35 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:35.199+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:09.390482+0000 front 2024-04-26T05:11:09.390548+0000 (oldest deadline 2024-04-26T05:11:34.090606+0000) 2024-04-26T05:11:35.492 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:35 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:35.222+0000 7fcc88b84700 -1 osd.5 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:00.724400+0000 front 2024-04-26T05:11:00.724363+0000 (oldest deadline 2024-04-26T05:11:24.824251+0000) 2024-04-26T05:11:35.492 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:35 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:35.222+0000 7fcc88b84700 -1 osd.5 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:11.225807+0000 front 2024-04-26T05:11:11.225760+0000 (oldest deadline 2024-04-26T05:11:33.525909+0000) 2024-04-26T05:11:35.991 INFO:journalctl@ceph.osd.4.smithi116.stdout:Apr 26 05:11:35 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-4[34852]: 2024-04-26T05:11:35.578+0000 7f9a53da3700 -1 osd.4 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.750291+0000 front 2024-04-26T05:11:03.750306+0000 (oldest deadline 2024-04-26T05:11:26.050247+0000) 2024-04-26T05:11:36.492 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:36 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:36.167+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:01.789439+0000 front 2024-04-26T05:11:01.789480+0000 (oldest deadline 2024-04-26T05:11:25.289408+0000) 2024-04-26T05:11:36.492 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:36 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:36.167+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:09.390482+0000 front 2024-04-26T05:11:09.390548+0000 (oldest deadline 2024-04-26T05:11:34.090606+0000) 2024-04-26T05:11:36.492 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:36 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:36.120+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.316461+0000 front 2024-04-26T05:11:03.316324+0000 (oldest deadline 2024-04-26T05:11:25.616201+0000) 2024-04-26T05:11:36.493 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:36 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:36.120+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:10.217682+0000 front 2024-04-26T05:11:10.217638+0000 (oldest deadline 2024-04-26T05:11:36.117550+0000) 2024-04-26T05:11:36.493 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:36 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:36.200+0000 7fcc88b84700 -1 osd.5 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:00.724400+0000 front 2024-04-26T05:11:00.724363+0000 (oldest deadline 2024-04-26T05:11:24.824251+0000) 2024-04-26T05:11:36.493 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:36 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:36.200+0000 7fcc88b84700 -1 osd.5 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:11.225807+0000 front 2024-04-26T05:11:11.225760+0000 (oldest deadline 2024-04-26T05:11:33.525909+0000) 2024-04-26T05:11:36.757 INFO:journalctl@ceph.osd.4.smithi116.stdout:Apr 26 05:11:36 smithi116 podman[95881]: 2024-04-26 05:11:36.538684689 +0000 UTC m=+5.471553542 container died 6b0bbf77cb39791eb4c37a15c387f74912293e06d25a8a23e0de5d505966d949 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-4, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=quincy-c61472c, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True) 2024-04-26T05:11:37.024 INFO:journalctl@ceph.osd.4.smithi116.stdout:Apr 26 05:11:36 smithi116 podman[95881]: 2024-04-26 05:11:36.921988879 +0000 UTC m=+5.854857729 container cleanup 6b0bbf77cb39791eb4c37a15c387f74912293e06d25a8a23e0de5d505966d949 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-4, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, ceph=True, CEPH_POINT_RELEASE=, RELEASE=quincy-c61472c, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:11:37.024 INFO:journalctl@ceph.osd.4.smithi116.stdout:Apr 26 05:11:36 smithi116 bash[95881]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-4 2024-04-26T05:11:37.491 INFO:journalctl@ceph.osd.4.smithi116.stdout:Apr 26 05:11:37 smithi116 podman[95900]: 2024-04-26 05:11:37.173371846 +0000 UTC m=+0.624128123 container remove 6b0bbf77cb39791eb4c37a15c387f74912293e06d25a8a23e0de5d505966d949 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-4, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-c61472c, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-04-26T05:11:37.492 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:37 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:37.179+0000 7fcc88b84700 -1 osd.5 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:00.724400+0000 front 2024-04-26T05:11:00.724363+0000 (oldest deadline 2024-04-26T05:11:24.824251+0000) 2024-04-26T05:11:37.492 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:37 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:37.179+0000 7fcc88b84700 -1 osd.5 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:11.225807+0000 front 2024-04-26T05:11:11.225760+0000 (oldest deadline 2024-04-26T05:11:33.525909+0000) 2024-04-26T05:11:37.492 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:37 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:37.080+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.316461+0000 front 2024-04-26T05:11:03.316324+0000 (oldest deadline 2024-04-26T05:11:25.616201+0000) 2024-04-26T05:11:37.492 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:37 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:37.080+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:10.217682+0000 front 2024-04-26T05:11:10.217638+0000 (oldest deadline 2024-04-26T05:11:36.117550+0000) 2024-04-26T05:11:37.493 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:37 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:37.139+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:01.789439+0000 front 2024-04-26T05:11:01.789480+0000 (oldest deadline 2024-04-26T05:11:25.289408+0000) 2024-04-26T05:11:37.493 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:37 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:37.140+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:09.390482+0000 front 2024-04-26T05:11:09.390548+0000 (oldest deadline 2024-04-26T05:11:34.090606+0000) 2024-04-26T05:11:37.770 INFO:journalctl@ceph.osd.4.smithi116.stdout:Apr 26 05:11:37 smithi116 podman[95999]: 2024-04-26 05:11:37.412432256 +0000 UTC m=+0.018224482 image pull 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17 2024-04-26T05:11:37.770 INFO:journalctl@ceph.osd.4.smithi116.stdout:Apr 26 05:11:37 smithi116 podman[95999]: 2024-04-26 05:11:37.632699614 +0000 UTC m=+0.238491822 container create 13ee3444a8a8bc612a805e6c5279cfbd80afc29145722ad5959fd7384440f707 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240424, RELEASE=quincy-c61472c, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, GIT_BRANCH=HEAD) 2024-04-26T05:11:38.100 INFO:journalctl@ceph.osd.4.smithi116.stdout:Apr 26 05:11:37 smithi116 podman[95999]: 2024-04-26 05:11:37.859985366 +0000 UTC m=+0.465777565 container init 13ee3444a8a8bc612a805e6c5279cfbd80afc29145722ad5959fd7384440f707 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=quincy-c61472c, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-04-26T05:11:38.100 INFO:journalctl@ceph.osd.4.smithi116.stdout:Apr 26 05:11:37 smithi116 podman[95999]: 2024-04-26 05:11:37.863169106 +0000 UTC m=+0.468961309 container start 13ee3444a8a8bc612a805e6c5279cfbd80afc29145722ad5959fd7384440f707 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=quincy-c61472c, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T05:11:38.100 INFO:journalctl@ceph.osd.4.smithi116.stdout:Apr 26 05:11:37 smithi116 podman[95999]: 2024-04-26 05:11:37.943291941 +0000 UTC m=+0.549084162 container attach 13ee3444a8a8bc612a805e6c5279cfbd80afc29145722ad5959fd7384440f707 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, RELEASE=quincy-c61472c, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-04-26T05:11:38.430 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:38 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:38.180+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:01.789439+0000 front 2024-04-26T05:11:01.789480+0000 (oldest deadline 2024-04-26T05:11:25.289408+0000) 2024-04-26T05:11:38.430 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:38 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:38.180+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:09.390482+0000 front 2024-04-26T05:11:09.390548+0000 (oldest deadline 2024-04-26T05:11:34.090606+0000) 2024-04-26T05:11:38.430 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:38 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:38.098+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.316461+0000 front 2024-04-26T05:11:03.316324+0000 (oldest deadline 2024-04-26T05:11:25.616201+0000) 2024-04-26T05:11:38.430 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:38 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:38.098+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:10.217682+0000 front 2024-04-26T05:11:10.217638+0000 (oldest deadline 2024-04-26T05:11:36.117550+0000) 2024-04-26T05:11:38.431 INFO:journalctl@ceph.osd.4.smithi116.stdout:Apr 26 05:11:38 smithi116 podman[95999]: 2024-04-26 05:11:38.148132201 +0000 UTC m=+0.753924408 container died 13ee3444a8a8bc612a805e6c5279cfbd80afc29145722ad5959fd7384440f707 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=quincy-c61472c, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-04-26T05:11:38.431 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:38 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:38.211+0000 7fcc88b84700 -1 osd.5 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:00.724400+0000 front 2024-04-26T05:11:00.724363+0000 (oldest deadline 2024-04-26T05:11:24.824251+0000) 2024-04-26T05:11:38.431 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:38 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:38.211+0000 7fcc88b84700 -1 osd.5 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:11.225807+0000 front 2024-04-26T05:11:11.225760+0000 (oldest deadline 2024-04-26T05:11:33.525909+0000) 2024-04-26T05:11:38.725 DEBUG:teuthology.orchestra.run.smithi116:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0b9350c6-0388-11ef-bc93-c7b262605968@osd.4.service' 2024-04-26T05:11:38.803 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:11:38.804 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-04-26T05:11:38.804 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-04-26T05:11:38.804 DEBUG:teuthology.orchestra.run.smithi116:> sudo systemctl stop ceph-0b9350c6-0388-11ef-bc93-c7b262605968@osd.5 2024-04-26T05:11:38.991 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:38 smithi116 systemd[1]: Stopping Ceph osd.5 for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T05:11:39.492 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:39 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:39.164+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:01.789439+0000 front 2024-04-26T05:11:01.789480+0000 (oldest deadline 2024-04-26T05:11:25.289408+0000) 2024-04-26T05:11:39.492 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:39 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:39.164+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:09.390482+0000 front 2024-04-26T05:11:09.390548+0000 (oldest deadline 2024-04-26T05:11:34.090606+0000) 2024-04-26T05:11:39.492 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:39 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:39.120+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.316461+0000 front 2024-04-26T05:11:03.316324+0000 (oldest deadline 2024-04-26T05:11:25.616201+0000) 2024-04-26T05:11:39.492 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:39 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:39.120+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:10.217682+0000 front 2024-04-26T05:11:10.217638+0000 (oldest deadline 2024-04-26T05:11:36.117550+0000) 2024-04-26T05:11:39.493 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:39 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:39.141+0000 7fcc8dd89700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:11:39.493 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:39 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:39.141+0000 7fcc8dd89700 -1 osd.5 100 *** Got signal Terminated *** 2024-04-26T05:11:39.493 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:39 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:39.141+0000 7fcc8dd89700 -1 osd.5 100 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:11:39.493 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:39 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:39.173+0000 7fcc88b84700 -1 osd.5 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:00.724400+0000 front 2024-04-26T05:11:00.724363+0000 (oldest deadline 2024-04-26T05:11:24.824251+0000) 2024-04-26T05:11:39.493 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:39 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:39.173+0000 7fcc88b84700 -1 osd.5 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:11.225807+0000 front 2024-04-26T05:11:11.225760+0000 (oldest deadline 2024-04-26T05:11:33.525909+0000) 2024-04-26T05:11:40.491 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:40 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:40.126+0000 7fcc88b84700 -1 osd.5 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:00.724400+0000 front 2024-04-26T05:11:00.724363+0000 (oldest deadline 2024-04-26T05:11:24.824251+0000) 2024-04-26T05:11:40.492 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:40 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:40.126+0000 7fcc88b84700 -1 osd.5 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:11.225807+0000 front 2024-04-26T05:11:11.225760+0000 (oldest deadline 2024-04-26T05:11:33.525909+0000) 2024-04-26T05:11:40.492 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:40 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:40.163+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.316461+0000 front 2024-04-26T05:11:03.316324+0000 (oldest deadline 2024-04-26T05:11:25.616201+0000) 2024-04-26T05:11:40.492 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:40 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:40.163+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:10.217682+0000 front 2024-04-26T05:11:10.217638+0000 (oldest deadline 2024-04-26T05:11:36.117550+0000) 2024-04-26T05:11:40.492 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:40 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:40.214+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:01.789439+0000 front 2024-04-26T05:11:01.789480+0000 (oldest deadline 2024-04-26T05:11:25.289408+0000) 2024-04-26T05:11:40.493 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:40 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:40.214+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:09.390482+0000 front 2024-04-26T05:11:09.390548+0000 (oldest deadline 2024-04-26T05:11:34.090606+0000) 2024-04-26T05:11:41.491 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:41 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:41.137+0000 7fcc88b84700 -1 osd.5 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:00.724400+0000 front 2024-04-26T05:11:00.724363+0000 (oldest deadline 2024-04-26T05:11:24.824251+0000) 2024-04-26T05:11:41.492 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:41 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:41.137+0000 7fcc88b84700 -1 osd.5 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:11.225807+0000 front 2024-04-26T05:11:11.225760+0000 (oldest deadline 2024-04-26T05:11:33.525909+0000) 2024-04-26T05:11:41.492 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:41 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:41.168+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:01.789439+0000 front 2024-04-26T05:11:01.789480+0000 (oldest deadline 2024-04-26T05:11:25.289408+0000) 2024-04-26T05:11:41.492 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:41 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:41.168+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:09.390482+0000 front 2024-04-26T05:11:09.390548+0000 (oldest deadline 2024-04-26T05:11:34.090606+0000) 2024-04-26T05:11:41.493 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:41 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:41.127+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.316461+0000 front 2024-04-26T05:11:03.316324+0000 (oldest deadline 2024-04-26T05:11:25.616201+0000) 2024-04-26T05:11:41.493 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:41 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:41.127+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:10.217682+0000 front 2024-04-26T05:11:10.217638+0000 (oldest deadline 2024-04-26T05:11:36.117550+0000) 2024-04-26T05:11:41.493 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:41 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:41.127+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-04-26T05:11:19.618575+0000 front 2024-04-26T05:11:19.618467+0000 (oldest deadline 2024-04-26T05:11:40.718316+0000) 2024-04-26T05:11:42.491 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:42 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:42.165+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:01.789439+0000 front 2024-04-26T05:11:01.789480+0000 (oldest deadline 2024-04-26T05:11:25.289408+0000) 2024-04-26T05:11:42.492 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:42 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:42.165+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:09.390482+0000 front 2024-04-26T05:11:09.390548+0000 (oldest deadline 2024-04-26T05:11:34.090606+0000) 2024-04-26T05:11:42.492 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:42 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:42.114+0000 7fcc88b84700 -1 osd.5 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:00.724400+0000 front 2024-04-26T05:11:00.724363+0000 (oldest deadline 2024-04-26T05:11:24.824251+0000) 2024-04-26T05:11:42.492 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:42 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:42.114+0000 7fcc88b84700 -1 osd.5 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:11.225807+0000 front 2024-04-26T05:11:11.225760+0000 (oldest deadline 2024-04-26T05:11:33.525909+0000) 2024-04-26T05:11:42.492 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:42 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:42.114+0000 7fcc88b84700 -1 osd.5 100 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-04-26T05:11:19.927360+0000 front 2024-04-26T05:11:19.927321+0000 (oldest deadline 2024-04-26T05:11:41.627226+0000) 2024-04-26T05:11:42.493 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:42 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:42.104+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.316461+0000 front 2024-04-26T05:11:03.316324+0000 (oldest deadline 2024-04-26T05:11:25.616201+0000) 2024-04-26T05:11:42.493 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:42 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:42.104+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:10.217682+0000 front 2024-04-26T05:11:10.217638+0000 (oldest deadline 2024-04-26T05:11:36.117550+0000) 2024-04-26T05:11:42.493 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:42 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:42.104+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-04-26T05:11:19.618575+0000 front 2024-04-26T05:11:19.618467+0000 (oldest deadline 2024-04-26T05:11:40.718316+0000) 2024-04-26T05:11:43.492 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:43 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:43.128+0000 7fcc88b84700 -1 osd.5 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:00.724400+0000 front 2024-04-26T05:11:00.724363+0000 (oldest deadline 2024-04-26T05:11:24.824251+0000) 2024-04-26T05:11:43.492 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:43 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:43.128+0000 7fcc88b84700 -1 osd.5 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:11.225807+0000 front 2024-04-26T05:11:11.225760+0000 (oldest deadline 2024-04-26T05:11:33.525909+0000) 2024-04-26T05:11:43.492 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:43 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:43.128+0000 7fcc88b84700 -1 osd.5 100 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-04-26T05:11:19.927360+0000 front 2024-04-26T05:11:19.927321+0000 (oldest deadline 2024-04-26T05:11:41.627226+0000) 2024-04-26T05:11:43.492 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:43 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:43.126+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:01.789439+0000 front 2024-04-26T05:11:01.789480+0000 (oldest deadline 2024-04-26T05:11:25.289408+0000) 2024-04-26T05:11:43.493 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:43 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:43.126+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:09.390482+0000 front 2024-04-26T05:11:09.390548+0000 (oldest deadline 2024-04-26T05:11:34.090606+0000) 2024-04-26T05:11:43.493 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:43 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:43.126+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-04-26T05:11:17.492113+0000 front 2024-04-26T05:11:17.492086+0000 (oldest deadline 2024-04-26T05:11:42.692364+0000) 2024-04-26T05:11:43.493 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:43 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:43.136+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.316461+0000 front 2024-04-26T05:11:03.316324+0000 (oldest deadline 2024-04-26T05:11:25.616201+0000) 2024-04-26T05:11:43.493 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:43 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:43.136+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:10.217682+0000 front 2024-04-26T05:11:10.217638+0000 (oldest deadline 2024-04-26T05:11:36.117550+0000) 2024-04-26T05:11:43.493 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:43 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:43.136+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-04-26T05:11:19.618575+0000 front 2024-04-26T05:11:19.618467+0000 (oldest deadline 2024-04-26T05:11:40.718316+0000) 2024-04-26T05:11:44.492 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:44 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:44.135+0000 7fcc88b84700 -1 osd.5 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:00.724400+0000 front 2024-04-26T05:11:00.724363+0000 (oldest deadline 2024-04-26T05:11:24.824251+0000) 2024-04-26T05:11:44.492 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:44 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:44.135+0000 7fcc88b84700 -1 osd.5 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:11.225807+0000 front 2024-04-26T05:11:11.225760+0000 (oldest deadline 2024-04-26T05:11:33.525909+0000) 2024-04-26T05:11:44.492 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:44 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5[40784]: 2024-04-26T05:11:44.135+0000 7fcc88b84700 -1 osd.5 100 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-04-26T05:11:19.927360+0000 front 2024-04-26T05:11:19.927321+0000 (oldest deadline 2024-04-26T05:11:41.627226+0000) 2024-04-26T05:11:44.492 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:44 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:44.104+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:01.789439+0000 front 2024-04-26T05:11:01.789480+0000 (oldest deadline 2024-04-26T05:11:25.289408+0000) 2024-04-26T05:11:44.492 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:44 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:44.104+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:09.390482+0000 front 2024-04-26T05:11:09.390548+0000 (oldest deadline 2024-04-26T05:11:34.090606+0000) 2024-04-26T05:11:44.493 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:44 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:44.104+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-04-26T05:11:17.492113+0000 front 2024-04-26T05:11:17.492086+0000 (oldest deadline 2024-04-26T05:11:42.692364+0000) 2024-04-26T05:11:44.493 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:44 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:44.185+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.316461+0000 front 2024-04-26T05:11:03.316324+0000 (oldest deadline 2024-04-26T05:11:25.616201+0000) 2024-04-26T05:11:44.493 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:44 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:44.185+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:10.217682+0000 front 2024-04-26T05:11:10.217638+0000 (oldest deadline 2024-04-26T05:11:36.117550+0000) 2024-04-26T05:11:44.493 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:44 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:44.185+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-04-26T05:11:19.618575+0000 front 2024-04-26T05:11:19.618467+0000 (oldest deadline 2024-04-26T05:11:40.718316+0000) 2024-04-26T05:11:44.763 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 05:11:44 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[81716]: ts=2024-04-26T05:11:44.276Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi116:8443/api/prometheus_receiver\": dial tcp 172.21.15.116:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi012.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.12:8443: connect: connection refused" 2024-04-26T05:11:44.763 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 05:11:44 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[81716]: ts=2024-04-26T05:11:44.277Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi012.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.12:8443: connect: connection refused" 2024-04-26T05:11:44.763 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 05:11:44 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[81716]: ts=2024-04-26T05:11:44.277Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi116:8443/api/prometheus_receiver\": dial tcp 172.21.15.116:8443: connect: connection refused" 2024-04-26T05:11:44.809 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:44 smithi116 podman[96108]: 2024-04-26 05:11:44.542577388 +0000 UTC m=+5.511556507 container died a551ef15fa7eacc813db255acbde13a71640618231a8bae872317a111de76bbe (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, RELEASE=quincy-c61472c, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-26T05:11:45.060 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:44 smithi116 podman[96108]: 2024-04-26 05:11:44.8999484 +0000 UTC m=+5.868927519 container cleanup a551ef15fa7eacc813db255acbde13a71640618231a8bae872317a111de76bbe (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5, RELEASE=quincy-c61472c, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:11:45.060 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:44 smithi116 bash[96108]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5 2024-04-26T05:11:45.493 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:45 smithi116 podman[96125]: 2024-04-26 05:11:45.192906441 +0000 UTC m=+0.645292118 container remove a551ef15fa7eacc813db255acbde13a71640618231a8bae872317a111de76bbe (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5, org.label-schema.schema-version=1.0, RELEASE=quincy-c61472c, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, io.buildah.version=1.33.5) 2024-04-26T05:11:45.494 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:45 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:45.233+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.316461+0000 front 2024-04-26T05:11:03.316324+0000 (oldest deadline 2024-04-26T05:11:25.616201+0000) 2024-04-26T05:11:45.494 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:45 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:45.233+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:10.217682+0000 front 2024-04-26T05:11:10.217638+0000 (oldest deadline 2024-04-26T05:11:36.117550+0000) 2024-04-26T05:11:45.494 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:45 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:45.233+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-04-26T05:11:19.618575+0000 front 2024-04-26T05:11:19.618467+0000 (oldest deadline 2024-04-26T05:11:40.718316+0000) 2024-04-26T05:11:45.495 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:45 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:45.058+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:01.789439+0000 front 2024-04-26T05:11:01.789480+0000 (oldest deadline 2024-04-26T05:11:25.289408+0000) 2024-04-26T05:11:45.495 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:45 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:45.058+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:09.390482+0000 front 2024-04-26T05:11:09.390548+0000 (oldest deadline 2024-04-26T05:11:34.090606+0000) 2024-04-26T05:11:45.495 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:45 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:45.058+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-04-26T05:11:17.492113+0000 front 2024-04-26T05:11:17.492086+0000 (oldest deadline 2024-04-26T05:11:42.692364+0000) 2024-04-26T05:11:45.821 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:45 smithi116 podman[96220]: 2024-04-26 05:11:45.418448122 +0000 UTC m=+0.017847229 image pull 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17 2024-04-26T05:11:45.821 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:45 smithi116 podman[96220]: 2024-04-26 05:11:45.526912932 +0000 UTC m=+0.126312032 container create 2f989164ce8138ee1cf5605ff65d05f1ae38c82f31a419b5f3b85136bc5d2c00 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, RELEASE=quincy-c61472c, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-04-26T05:11:45.821 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:45 smithi116 podman[96220]: 2024-04-26 05:11:45.738309522 +0000 UTC m=+0.337708628 container init 2f989164ce8138ee1cf5605ff65d05f1ae38c82f31a419b5f3b85136bc5d2c00 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5-deactivate, RELEASE=quincy-c61472c, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-04-26T05:11:45.822 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:45 smithi116 podman[96220]: 2024-04-26 05:11:45.74162371 +0000 UTC m=+0.341022809 container start 2f989164ce8138ee1cf5605ff65d05f1ae38c82f31a419b5f3b85136bc5d2c00 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=quincy-c61472c, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-26T05:11:45.822 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:45 smithi116 podman[96220]: 2024-04-26 05:11:45.821050625 +0000 UTC m=+0.420449738 container attach 2f989164ce8138ee1cf5605ff65d05f1ae38c82f31a419b5f3b85136bc5d2c00 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=quincy-c61472c, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424) 2024-04-26T05:11:46.095 INFO:journalctl@ceph.osd.5.smithi116.stdout:Apr 26 05:11:46 smithi116 podman[96220]: 2024-04-26 05:11:46.020010866 +0000 UTC m=+0.619409973 container died 2f989164ce8138ee1cf5605ff65d05f1ae38c82f31a419b5f3b85136bc5d2c00 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, RELEASE=quincy-c61472c, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-26T05:11:46.492 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:46 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:46.214+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.316461+0000 front 2024-04-26T05:11:03.316324+0000 (oldest deadline 2024-04-26T05:11:25.616201+0000) 2024-04-26T05:11:46.492 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:46 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:46.214+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:10.217682+0000 front 2024-04-26T05:11:10.217638+0000 (oldest deadline 2024-04-26T05:11:36.117550+0000) 2024-04-26T05:11:46.492 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:46 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:46.214+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-04-26T05:11:19.618575+0000 front 2024-04-26T05:11:19.618467+0000 (oldest deadline 2024-04-26T05:11:40.718316+0000) 2024-04-26T05:11:46.492 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:46 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:46.092+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:01.789439+0000 front 2024-04-26T05:11:01.789480+0000 (oldest deadline 2024-04-26T05:11:25.289408+0000) 2024-04-26T05:11:46.492 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:46 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:46.092+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:09.390482+0000 front 2024-04-26T05:11:09.390548+0000 (oldest deadline 2024-04-26T05:11:34.090606+0000) 2024-04-26T05:11:46.493 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:46 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:46.092+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-04-26T05:11:17.492113+0000 front 2024-04-26T05:11:17.492086+0000 (oldest deadline 2024-04-26T05:11:42.692364+0000) 2024-04-26T05:11:46.653 DEBUG:teuthology.orchestra.run.smithi116:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0b9350c6-0388-11ef-bc93-c7b262605968@osd.5.service' 2024-04-26T05:11:46.693 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:11:46.693 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-04-26T05:11:46.694 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-04-26T05:11:46.694 DEBUG:teuthology.orchestra.run.smithi116:> sudo systemctl stop ceph-0b9350c6-0388-11ef-bc93-c7b262605968@osd.6 2024-04-26T05:11:46.992 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:46 smithi116 systemd[1]: Stopping Ceph osd.6 for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T05:11:47.476 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:47 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:47.242+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.316461+0000 front 2024-04-26T05:11:03.316324+0000 (oldest deadline 2024-04-26T05:11:25.616201+0000) 2024-04-26T05:11:47.476 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:47 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:47.242+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:10.217682+0000 front 2024-04-26T05:11:10.217638+0000 (oldest deadline 2024-04-26T05:11:36.117550+0000) 2024-04-26T05:11:47.476 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:47 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:47.242+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-04-26T05:11:19.618575+0000 front 2024-04-26T05:11:19.618467+0000 (oldest deadline 2024-04-26T05:11:40.718316+0000) 2024-04-26T05:11:47.476 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:47 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:47.019+0000 7fe122427700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:11:47.477 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:47 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:47.019+0000 7fe122427700 -1 osd.6 100 *** Got signal Terminated *** 2024-04-26T05:11:47.477 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:47 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:47.019+0000 7fe122427700 -1 osd.6 100 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:11:47.477 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:47 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:47.118+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:01.789439+0000 front 2024-04-26T05:11:01.789480+0000 (oldest deadline 2024-04-26T05:11:25.289408+0000) 2024-04-26T05:11:47.477 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:47 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:47.118+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:09.390482+0000 front 2024-04-26T05:11:09.390548+0000 (oldest deadline 2024-04-26T05:11:34.090606+0000) 2024-04-26T05:11:47.477 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:47 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:47.118+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-04-26T05:11:17.492113+0000 front 2024-04-26T05:11:17.492086+0000 (oldest deadline 2024-04-26T05:11:42.692364+0000) 2024-04-26T05:11:48.491 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:48.120+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:01.789439+0000 front 2024-04-26T05:11:01.789480+0000 (oldest deadline 2024-04-26T05:11:25.289408+0000) 2024-04-26T05:11:48.492 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:48.120+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:09.390482+0000 front 2024-04-26T05:11:09.390548+0000 (oldest deadline 2024-04-26T05:11:34.090606+0000) 2024-04-26T05:11:48.492 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:48.120+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-04-26T05:11:17.492113+0000 front 2024-04-26T05:11:17.492086+0000 (oldest deadline 2024-04-26T05:11:42.692364+0000) 2024-04-26T05:11:48.492 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:48.204+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.316461+0000 front 2024-04-26T05:11:03.316324+0000 (oldest deadline 2024-04-26T05:11:25.616201+0000) 2024-04-26T05:11:48.492 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:48.204+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:10.217682+0000 front 2024-04-26T05:11:10.217638+0000 (oldest deadline 2024-04-26T05:11:36.117550+0000) 2024-04-26T05:11:48.492 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:48 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:48.204+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-04-26T05:11:19.618575+0000 front 2024-04-26T05:11:19.618467+0000 (oldest deadline 2024-04-26T05:11:40.718316+0000) 2024-04-26T05:11:49.491 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:49 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:49.112+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:01.789439+0000 front 2024-04-26T05:11:01.789480+0000 (oldest deadline 2024-04-26T05:11:25.289408+0000) 2024-04-26T05:11:49.492 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:49 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:49.112+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:09.390482+0000 front 2024-04-26T05:11:09.390548+0000 (oldest deadline 2024-04-26T05:11:34.090606+0000) 2024-04-26T05:11:49.492 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:49 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:49.112+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-04-26T05:11:17.492113+0000 front 2024-04-26T05:11:17.492086+0000 (oldest deadline 2024-04-26T05:11:42.692364+0000) 2024-04-26T05:11:49.492 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:49 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:49.173+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.316461+0000 front 2024-04-26T05:11:03.316324+0000 (oldest deadline 2024-04-26T05:11:25.616201+0000) 2024-04-26T05:11:49.492 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:49 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:49.173+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:10.217682+0000 front 2024-04-26T05:11:10.217638+0000 (oldest deadline 2024-04-26T05:11:36.117550+0000) 2024-04-26T05:11:49.492 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:49 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:49.173+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-04-26T05:11:19.618575+0000 front 2024-04-26T05:11:19.618467+0000 (oldest deadline 2024-04-26T05:11:40.718316+0000) 2024-04-26T05:11:50.492 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:50 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:50.125+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:01.789439+0000 front 2024-04-26T05:11:01.789480+0000 (oldest deadline 2024-04-26T05:11:25.289408+0000) 2024-04-26T05:11:50.492 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:50 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:50.125+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:09.390482+0000 front 2024-04-26T05:11:09.390548+0000 (oldest deadline 2024-04-26T05:11:34.090606+0000) 2024-04-26T05:11:50.492 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:50 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:50.125+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-04-26T05:11:17.492113+0000 front 2024-04-26T05:11:17.492086+0000 (oldest deadline 2024-04-26T05:11:42.692364+0000) 2024-04-26T05:11:50.492 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:50 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:50.125+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6830 osd.3 since back 2024-04-26T05:11:22.692840+0000 front 2024-04-26T05:11:22.692807+0000 (oldest deadline 2024-04-26T05:11:49.693219+0000) 2024-04-26T05:11:50.493 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:50 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:50.210+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.316461+0000 front 2024-04-26T05:11:03.316324+0000 (oldest deadline 2024-04-26T05:11:25.616201+0000) 2024-04-26T05:11:50.493 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:50 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:50.210+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:10.217682+0000 front 2024-04-26T05:11:10.217638+0000 (oldest deadline 2024-04-26T05:11:36.117550+0000) 2024-04-26T05:11:50.493 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:50 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:50.210+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-04-26T05:11:19.618575+0000 front 2024-04-26T05:11:19.618467+0000 (oldest deadline 2024-04-26T05:11:40.718316+0000) 2024-04-26T05:11:50.493 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:50 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:50.210+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6830 osd.3 since back 2024-04-26T05:11:27.719736+0000 front 2024-04-26T05:11:27.719782+0000 (oldest deadline 2024-04-26T05:11:50.019708+0000) 2024-04-26T05:11:51.492 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:51 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:51.120+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:01.789439+0000 front 2024-04-26T05:11:01.789480+0000 (oldest deadline 2024-04-26T05:11:25.289408+0000) 2024-04-26T05:11:51.492 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:51 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:51.120+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:09.390482+0000 front 2024-04-26T05:11:09.390548+0000 (oldest deadline 2024-04-26T05:11:34.090606+0000) 2024-04-26T05:11:51.492 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:51 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:51.120+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-04-26T05:11:17.492113+0000 front 2024-04-26T05:11:17.492086+0000 (oldest deadline 2024-04-26T05:11:42.692364+0000) 2024-04-26T05:11:51.492 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:51 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6[46805]: 2024-04-26T05:11:51.120+0000 7fe11ba1f700 -1 osd.6 100 heartbeat_check: no reply from 172.21.15.12:6830 osd.3 since back 2024-04-26T05:11:22.692840+0000 front 2024-04-26T05:11:22.692807+0000 (oldest deadline 2024-04-26T05:11:49.693219+0000) 2024-04-26T05:11:51.492 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:51 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:51.241+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.316461+0000 front 2024-04-26T05:11:03.316324+0000 (oldest deadline 2024-04-26T05:11:25.616201+0000) 2024-04-26T05:11:51.492 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:51 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:51.241+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:10.217682+0000 front 2024-04-26T05:11:10.217638+0000 (oldest deadline 2024-04-26T05:11:36.117550+0000) 2024-04-26T05:11:51.493 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:51 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:51.241+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-04-26T05:11:19.618575+0000 front 2024-04-26T05:11:19.618467+0000 (oldest deadline 2024-04-26T05:11:40.718316+0000) 2024-04-26T05:11:51.493 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:51 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:51.241+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6830 osd.3 since back 2024-04-26T05:11:27.719736+0000 front 2024-04-26T05:11:27.719782+0000 (oldest deadline 2024-04-26T05:11:50.019708+0000) 2024-04-26T05:11:52.637 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:52 smithi116 podman[96326]: 2024-04-26 05:11:52.358556719 +0000 UTC m=+5.447037652 container died cdc0eb6f4fa290838bd6e35de94e3a2c2a92bb93e91a57cd8b8a3800bc52d624 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=quincy-c61472c, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-04-26T05:11:52.637 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:52 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:52.287+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.316461+0000 front 2024-04-26T05:11:03.316324+0000 (oldest deadline 2024-04-26T05:11:25.616201+0000) 2024-04-26T05:11:52.637 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:52 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:52.287+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:10.217682+0000 front 2024-04-26T05:11:10.217638+0000 (oldest deadline 2024-04-26T05:11:36.117550+0000) 2024-04-26T05:11:52.638 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:52 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:52.287+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-04-26T05:11:19.618575+0000 front 2024-04-26T05:11:19.618467+0000 (oldest deadline 2024-04-26T05:11:40.718316+0000) 2024-04-26T05:11:52.638 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:52 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:52.287+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6830 osd.3 since back 2024-04-26T05:11:27.719736+0000 front 2024-04-26T05:11:27.719782+0000 (oldest deadline 2024-04-26T05:11:50.019708+0000) 2024-04-26T05:11:52.970 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:52 smithi116 podman[96326]: 2024-04-26 05:11:52.719065983 +0000 UTC m=+5.807546900 container cleanup cdc0eb6f4fa290838bd6e35de94e3a2c2a92bb93e91a57cd8b8a3800bc52d624 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-c61472c, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-04-26T05:11:52.970 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:52 smithi116 bash[96326]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6 2024-04-26T05:11:53.241 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:52 smithi116 podman[96343]: 2024-04-26 05:11:52.970092672 +0000 UTC m=+0.614282618 container remove cdc0eb6f4fa290838bd6e35de94e3a2c2a92bb93e91a57cd8b8a3800bc52d624 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6, GIT_CLEAN=True, RELEASE=quincy-c61472c, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-04-26T05:11:53.526 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:53 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:53.273+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.316461+0000 front 2024-04-26T05:11:03.316324+0000 (oldest deadline 2024-04-26T05:11:25.616201+0000) 2024-04-26T05:11:53.527 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:53 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:53.273+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:10.217682+0000 front 2024-04-26T05:11:10.217638+0000 (oldest deadline 2024-04-26T05:11:36.117550+0000) 2024-04-26T05:11:53.527 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:53 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:53.273+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-04-26T05:11:19.618575+0000 front 2024-04-26T05:11:19.618467+0000 (oldest deadline 2024-04-26T05:11:40.718316+0000) 2024-04-26T05:11:53.527 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:53 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:53.273+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6830 osd.3 since back 2024-04-26T05:11:27.719736+0000 front 2024-04-26T05:11:27.719782+0000 (oldest deadline 2024-04-26T05:11:50.019708+0000) 2024-04-26T05:11:53.527 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:53 smithi116 podman[96439]: 2024-04-26 05:11:53.188943524 +0000 UTC m=+0.018069804 image pull 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17 2024-04-26T05:11:53.527 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:53 smithi116 podman[96439]: 2024-04-26 05:11:53.29583612 +0000 UTC m=+0.124962372 container create 7cb9b4d81bfba84e3a967b9a414cb3ecb6969041896127a1e8d79023ed074751 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6-deactivate, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=quincy-c61472c, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:11:53.527 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:53 smithi116 podman[96439]: 2024-04-26 05:11:53.523365592 +0000 UTC m=+0.352491850 container init 7cb9b4d81bfba84e3a967b9a414cb3ecb6969041896127a1e8d79023ed074751 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, RELEASE=quincy-c61472c, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-26T05:11:53.528 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:53 smithi116 podman[96439]: 2024-04-26 05:11:53.526401744 +0000 UTC m=+0.355527997 container start 7cb9b4d81bfba84e3a967b9a414cb3ecb6969041896127a1e8d79023ed074751 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6-deactivate, RELEASE=quincy-c61472c, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-26T05:11:53.802 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:53 smithi116 podman[96439]: 2024-04-26 05:11:53.631659198 +0000 UTC m=+0.460785460 container attach 7cb9b4d81bfba84e3a967b9a414cb3ecb6969041896127a1e8d79023ed074751 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, RELEASE=quincy-c61472c, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-04-26T05:11:54.118 INFO:journalctl@ceph.osd.6.smithi116.stdout:Apr 26 05:11:53 smithi116 podman[96439]: 2024-04-26 05:11:53.801773484 +0000 UTC m=+0.630899739 container died 7cb9b4d81bfba84e3a967b9a414cb3ecb6969041896127a1e8d79023ed074751 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-6-deactivate, GIT_CLEAN=True, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=quincy-c61472c, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-26T05:11:54.410 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:54 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:54.312+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.316461+0000 front 2024-04-26T05:11:03.316324+0000 (oldest deadline 2024-04-26T05:11:25.616201+0000) 2024-04-26T05:11:54.410 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:54 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:54.312+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:10.217682+0000 front 2024-04-26T05:11:10.217638+0000 (oldest deadline 2024-04-26T05:11:36.117550+0000) 2024-04-26T05:11:54.410 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:54 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:54.312+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-04-26T05:11:19.618575+0000 front 2024-04-26T05:11:19.618467+0000 (oldest deadline 2024-04-26T05:11:40.718316+0000) 2024-04-26T05:11:54.410 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:54 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:54.312+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6830 osd.3 since back 2024-04-26T05:11:27.719736+0000 front 2024-04-26T05:11:27.719782+0000 (oldest deadline 2024-04-26T05:11:50.019708+0000) 2024-04-26T05:11:54.430 DEBUG:teuthology.orchestra.run.smithi116:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0b9350c6-0388-11ef-bc93-c7b262605968@osd.6.service' 2024-04-26T05:11:54.468 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:11:54.468 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-04-26T05:11:54.469 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-04-26T05:11:54.469 DEBUG:teuthology.orchestra.run.smithi116:> sudo systemctl stop ceph-0b9350c6-0388-11ef-bc93-c7b262605968@osd.7 2024-04-26T05:11:54.674 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 05:11:54 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[81716]: ts=2024-04-26T05:11:54.277Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi012.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.12:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi116:8443/api/prometheus_receiver\": dial tcp 172.21.15.116:8443: connect: connection refused" 2024-04-26T05:11:54.674 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 05:11:54 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[81716]: ts=2024-04-26T05:11:54.278Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi012.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.12:8443: connect: connection refused" 2024-04-26T05:11:54.675 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 05:11:54 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[81716]: ts=2024-04-26T05:11:54.278Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi116:8443/api/prometheus_receiver\": dial tcp 172.21.15.116:8443: connect: connection refused" 2024-04-26T05:11:54.741 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:54 smithi116 systemd[1]: Stopping Ceph osd.7 for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T05:11:55.241 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:54 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:54.787+0000 7f8d91921700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:11:55.242 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:54 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:54.787+0000 7f8d91921700 -1 osd.7 100 *** Got signal Terminated *** 2024-04-26T05:11:55.242 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:54 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:54.787+0000 7f8d91921700 -1 osd.7 100 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:11:55.742 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:55 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:55.283+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.316461+0000 front 2024-04-26T05:11:03.316324+0000 (oldest deadline 2024-04-26T05:11:25.616201+0000) 2024-04-26T05:11:55.742 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:55 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:55.283+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:10.217682+0000 front 2024-04-26T05:11:10.217638+0000 (oldest deadline 2024-04-26T05:11:36.117550+0000) 2024-04-26T05:11:55.742 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:55 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:55.283+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-04-26T05:11:19.618575+0000 front 2024-04-26T05:11:19.618467+0000 (oldest deadline 2024-04-26T05:11:40.718316+0000) 2024-04-26T05:11:55.742 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:55 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:55.283+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6830 osd.3 since back 2024-04-26T05:11:27.719736+0000 front 2024-04-26T05:11:27.719782+0000 (oldest deadline 2024-04-26T05:11:50.019708+0000) 2024-04-26T05:11:56.491 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:56 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:56.263+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.316461+0000 front 2024-04-26T05:11:03.316324+0000 (oldest deadline 2024-04-26T05:11:25.616201+0000) 2024-04-26T05:11:56.492 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:56 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:56.263+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:10.217682+0000 front 2024-04-26T05:11:10.217638+0000 (oldest deadline 2024-04-26T05:11:36.117550+0000) 2024-04-26T05:11:56.492 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:56 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:56.263+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-04-26T05:11:19.618575+0000 front 2024-04-26T05:11:19.618467+0000 (oldest deadline 2024-04-26T05:11:40.718316+0000) 2024-04-26T05:11:56.492 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:56 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:56.263+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6830 osd.3 since back 2024-04-26T05:11:27.719736+0000 front 2024-04-26T05:11:27.719782+0000 (oldest deadline 2024-04-26T05:11:50.019708+0000) 2024-04-26T05:11:56.492 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:11:56 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:11:56.045Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.12:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.12:8765: connect: connection refused" 2024-04-26T05:11:56.493 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:11:56 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:11:56.045Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.12:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.12:8765: connect: connection refused" 2024-04-26T05:11:56.493 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:11:56 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:11:56.048Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.12:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.12:8765: connect: connection refused" 2024-04-26T05:11:56.493 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:11:56 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:11:56.049Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.12:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.12:8765: connect: connection refused" 2024-04-26T05:11:56.493 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:11:56 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:11:56.049Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.12:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.12:8765: connect: connection refused" 2024-04-26T05:11:57.742 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:57 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:57.253+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.316461+0000 front 2024-04-26T05:11:03.316324+0000 (oldest deadline 2024-04-26T05:11:25.616201+0000) 2024-04-26T05:11:57.742 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:57 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:57.253+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:10.217682+0000 front 2024-04-26T05:11:10.217638+0000 (oldest deadline 2024-04-26T05:11:36.117550+0000) 2024-04-26T05:11:57.742 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:57 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:57.253+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-04-26T05:11:19.618575+0000 front 2024-04-26T05:11:19.618467+0000 (oldest deadline 2024-04-26T05:11:40.718316+0000) 2024-04-26T05:11:57.742 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:57 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:57.253+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6830 osd.3 since back 2024-04-26T05:11:27.719736+0000 front 2024-04-26T05:11:27.719782+0000 (oldest deadline 2024-04-26T05:11:50.019708+0000) 2024-04-26T05:11:58.741 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:58 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:58.286+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.316461+0000 front 2024-04-26T05:11:03.316324+0000 (oldest deadline 2024-04-26T05:11:25.616201+0000) 2024-04-26T05:11:58.742 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:58 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:58.286+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:10.217682+0000 front 2024-04-26T05:11:10.217638+0000 (oldest deadline 2024-04-26T05:11:36.117550+0000) 2024-04-26T05:11:58.742 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:58 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:58.286+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-04-26T05:11:19.618575+0000 front 2024-04-26T05:11:19.618467+0000 (oldest deadline 2024-04-26T05:11:40.718316+0000) 2024-04-26T05:11:58.742 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:58 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:58.286+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6830 osd.3 since back 2024-04-26T05:11:27.719736+0000 front 2024-04-26T05:11:27.719782+0000 (oldest deadline 2024-04-26T05:11:50.019708+0000) 2024-04-26T05:11:59.741 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:59 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:59.312+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-04-26T05:11:03.316461+0000 front 2024-04-26T05:11:03.316324+0000 (oldest deadline 2024-04-26T05:11:25.616201+0000) 2024-04-26T05:11:59.742 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:59 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:59.312+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-04-26T05:11:10.217682+0000 front 2024-04-26T05:11:10.217638+0000 (oldest deadline 2024-04-26T05:11:36.117550+0000) 2024-04-26T05:11:59.742 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:59 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:59.312+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-04-26T05:11:19.618575+0000 front 2024-04-26T05:11:19.618467+0000 (oldest deadline 2024-04-26T05:11:40.718316+0000) 2024-04-26T05:11:59.742 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:11:59 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7[52446]: 2024-04-26T05:11:59.312+0000 7f8d8bf1b700 -1 osd.7 100 heartbeat_check: no reply from 172.21.15.12:6830 osd.3 since back 2024-04-26T05:11:27.719736+0000 front 2024-04-26T05:11:27.719782+0000 (oldest deadline 2024-04-26T05:11:50.019708+0000) 2024-04-26T05:12:00.472 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:12:00 smithi116 podman[96546]: 2024-04-26 05:12:00.203357583 +0000 UTC m=+5.514232496 container died 3f6689213fdae8770a0d9b3042b72e88f1fcea3c3d8d710a1ba8d5b0fb7c680d (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-c61472c, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-26T05:12:00.741 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:12:00 smithi116 podman[96546]: 2024-04-26 05:12:00.562401053 +0000 UTC m=+5.873275965 container cleanup 3f6689213fdae8770a0d9b3042b72e88f1fcea3c3d8d710a1ba8d5b0fb7c680d (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, RELEASE=quincy-c61472c, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-04-26T05:12:00.742 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:12:00 smithi116 bash[96546]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7 2024-04-26T05:12:01.173 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:12:00 smithi116 podman[96563]: 2024-04-26 05:12:00.83021674 +0000 UTC m=+0.621288668 container remove 3f6689213fdae8770a0d9b3042b72e88f1fcea3c3d8d710a1ba8d5b0fb7c680d (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=quincy-c61472c, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-26T05:12:01.486 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:12:01 smithi116 podman[96656]: 2024-04-26 05:12:01.075617412 +0000 UTC m=+0.018587080 image pull 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17 2024-04-26T05:12:01.486 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:12:01 smithi116 podman[96656]: 2024-04-26 05:12:01.172756286 +0000 UTC m=+0.115725997 container create b253ed807b86538363d8392a345463d59aac03742b474a154e75b2645df23329 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, ceph=True, CEPH_POINT_RELEASE=, RELEASE=quincy-c61472c) 2024-04-26T05:12:01.486 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:12:01 smithi116 podman[96656]: 2024-04-26 05:12:01.408509902 +0000 UTC m=+0.351479571 container init b253ed807b86538363d8392a345463d59aac03742b474a154e75b2645df23329 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, RELEASE=quincy-c61472c, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-04-26T05:12:01.487 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:12:01 smithi116 podman[96656]: 2024-04-26 05:12:01.411896047 +0000 UTC m=+0.354865717 container start b253ed807b86538363d8392a345463d59aac03742b474a154e75b2645df23329 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=quincy-c61472c, ceph=True, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2) 2024-04-26T05:12:01.487 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:12:01 smithi116 podman[96656]: 2024-04-26 05:12:01.483525782 +0000 UTC m=+0.426495452 container attach b253ed807b86538363d8392a345463d59aac03742b474a154e75b2645df23329 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, RELEASE=quincy-c61472c, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-04-26T05:12:01.741 INFO:journalctl@ceph.osd.7.smithi116.stdout:Apr 26 05:12:01 smithi116 podman[96656]: 2024-04-26 05:12:01.691075161 +0000 UTC m=+0.634044830 container died b253ed807b86538363d8392a345463d59aac03742b474a154e75b2645df23329 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=quincy-c61472c, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-04-26T05:12:02.264 DEBUG:teuthology.orchestra.run.smithi116:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0b9350c6-0388-11ef-bc93-c7b262605968@osd.7.service' 2024-04-26T05:12:02.303 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:12:02.303 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-04-26T05:12:02.303 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-04-26T05:12:02.304 DEBUG:teuthology.orchestra.run.smithi116:> sudo systemctl stop ceph-0b9350c6-0388-11ef-bc93-c7b262605968@prometheus.a 2024-04-26T05:12:02.615 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:12:02 smithi116 systemd[1]: Stopping Ceph prometheus.a for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T05:12:02.891 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:12:02 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:12:02.614Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-26T05:12:02.892 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:12:02 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:12:02.615Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-26T05:12:02.892 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:12:02 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:12:02.615Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-26T05:12:02.892 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:12:02 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:12:02.615Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-26T05:12:02.893 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:12:02 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:12:02.615Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-26T05:12:02.893 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:12:02 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:12:02.615Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-26T05:12:02.893 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:12:02 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:12:02.615Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-26T05:12:02.893 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:12:02 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:12:02.615Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-26T05:12:02.895 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:12:02 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:12:02.615Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-26T05:12:02.895 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:12:02 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:12:02.615Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-26T05:12:02.895 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:12:02 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:12:02.615Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-26T05:12:02.895 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:12:02 smithi116 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a[75257]: ts=2024-04-26T05:12:02.615Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-26T05:12:02.895 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:12:02 smithi116 podman[96762]: 2024-04-26 05:12:02.625103075 +0000 UTC m=+0.100820526 container died e9e5b93523ae1fbdc6028d790c32812e1be51d62d02337a9aece4bf3d215a4e5 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T05:12:03.240 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:12:02 smithi116 podman[96762]: 2024-04-26 05:12:02.98123105 +0000 UTC m=+0.456948505 container cleanup e9e5b93523ae1fbdc6028d790c32812e1be51d62d02337a9aece4bf3d215a4e5 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T05:12:03.241 INFO:journalctl@ceph.prometheus.a.smithi116.stdout:Apr 26 05:12:02 smithi116 bash[96762]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968-prometheus-a 2024-04-26T05:12:03.397 DEBUG:teuthology.orchestra.run.smithi116:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0b9350c6-0388-11ef-bc93-c7b262605968@prometheus.a.service' 2024-04-26T05:12:03.430 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:12:03.430 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-04-26T05:12:03.431 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-26T05:12:04.468 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 05:12:04 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[81716]: ts=2024-04-26T05:12:04.277Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi116:8443/api/prometheus_receiver\": dial tcp 172.21.15.116:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi012.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.12:8443: connect: connection refused" 2024-04-26T05:12:04.468 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 05:12:04 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[81716]: ts=2024-04-26T05:12:04.279Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi012.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.12:8443: connect: connection refused" 2024-04-26T05:12:04.468 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 05:12:04 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[81716]: ts=2024-04-26T05:12:04.279Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi116:8443/api/prometheus_receiver\": dial tcp 172.21.15.116:8443: connect: connection refused" 2024-04-26T05:12:05.253 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 05:12:04 smithi012 systemd[1]: Stopping Ceph alertmanager.a for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T05:12:05.253 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 05:12:05 smithi012 ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a[81716]: ts=2024-04-26T05:12:05.252Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-26T05:12:05.513 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 05:12:05 smithi012 podman[139470]: 2024-04-26 05:12:05.304221134 +0000 UTC m=+0.159707632 container died 217bb19e153b57d1d56bfdea3f7251d164d284f57bca979ba20ae14265006d8a (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T05:12:05.991 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 05:12:05 smithi012 podman[139470]: 2024-04-26 05:12:05.77002411 +0000 UTC m=+0.625510664 container cleanup 217bb19e153b57d1d56bfdea3f7251d164d284f57bca979ba20ae14265006d8a (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T05:12:05.991 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 05:12:05 smithi012 bash[139470]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a 2024-04-26T05:12:06.255 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 05:12:06 smithi012 podman[139482]: 2024-04-26 05:12:06.179835533 +0000 UTC m=+0.872241938 container remove 217bb19e153b57d1d56bfdea3f7251d164d284f57bca979ba20ae14265006d8a (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T05:12:06.255 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 05:12:06 smithi012 podman[139482]: 2024-04-26 05:12:06.255347605 +0000 UTC m=+0.947754010 volume remove a341212677031dae9e6b64a6b2a13d60cf3e420e820a4ffead8a01fcd4fc8daf 2024-04-26T05:12:06.513 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 05:12:06 smithi012 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-26T05:12:06.513 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 05:12:06 smithi012 systemd[1]: Stopped Ceph alertmanager.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T05:12:06.513 INFO:journalctl@ceph.alertmanager.a.smithi012.stdout:Apr 26 05:12:06 smithi012 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@alertmanager.a.service: Consumed 1.730s CPU time. 2024-04-26T05:12:07.013 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 05:12:06 smithi012 systemd[1]: Stopping Ceph node-exporter.a for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T05:12:07.013 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 05:12:06 smithi012 podman[139592]: 2024-04-26 05:12:06.818776982 +0000 UTC m=+0.096115167 container died 6390338999c1df79904119d62135930f153726a0f654122e1655880d620a1621 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T05:12:07.434 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 05:12:07 smithi012 podman[139592]: 2024-04-26 05:12:07.208079612 +0000 UTC m=+0.485417793 container cleanup 6390338999c1df79904119d62135930f153726a0f654122e1655880d620a1621 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T05:12:07.434 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 05:12:07 smithi012 bash[139592]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a 2024-04-26T05:12:07.745 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 05:12:07 smithi012 podman[139604]: 2024-04-26 05:12:07.434210314 +0000 UTC m=+0.615107977 container remove 6390338999c1df79904119d62135930f153726a0f654122e1655880d620a1621 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T05:12:07.745 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 05:12:07 smithi012 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-26T05:12:07.745 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 05:12:07 smithi012 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-26T05:12:07.746 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 05:12:07 smithi012 systemd[1]: Stopped Ceph node-exporter.a for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T05:12:07.746 INFO:journalctl@ceph.node-exporter.a.smithi012.stdout:Apr 26 05:12:07 smithi012 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@node-exporter.a.service: Consumed 2.234s CPU time. 2024-04-26T05:12:15.126 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-26T05:12:17.241 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 05:12:16 smithi116 systemd[1]: Stopping Ceph node-exporter.b for 0b9350c6-0388-11ef-bc93-c7b262605968... 2024-04-26T05:12:17.241 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 05:12:17 smithi116 podman[97224]: 2024-04-26 05:12:17.062685097 +0000 UTC m=+0.105640819 container died 7d0b1e809531c495421cfadd77ef13b3c561449f6a88f0fb881426fb653c5704 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T05:12:17.569 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 05:12:17 smithi116 podman[97224]: 2024-04-26 05:12:17.368519306 +0000 UTC m=+0.411475040 container cleanup 7d0b1e809531c495421cfadd77ef13b3c561449f6a88f0fb881426fb653c5704 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T05:12:17.570 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 05:12:17 smithi116 bash[97224]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b 2024-04-26T05:12:17.981 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 05:12:17 smithi116 podman[97238]: 2024-04-26 05:12:17.56951242 +0000 UTC m=+0.510949169 container remove 7d0b1e809531c495421cfadd77ef13b3c561449f6a88f0fb881426fb653c5704 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-0b9350c6-0388-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T05:12:17.981 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 05:12:17 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-26T05:12:17.981 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 05:12:17 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-26T05:12:17.981 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 05:12:17 smithi116 systemd[1]: Stopped Ceph node-exporter.b for 0b9350c6-0388-11ef-bc93-c7b262605968. 2024-04-26T05:12:17.981 INFO:journalctl@ceph.node-exporter.b.smithi116.stdout:Apr 26 05:12:17 smithi116 systemd[1]: ceph-0b9350c6-0388-11ef-bc93-c7b262605968@node-exporter.b.service: Consumed 2.056s CPU time. 2024-04-26T05:12:21.249 DEBUG:teuthology.orchestra.run.smithi012:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-26T05:12:21.281 DEBUG:teuthology.orchestra.run.smithi116:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-26T05:12:21.307 INFO:tasks.cephadm:Archiving crash dumps... 2024-04-26T05:12:21.309 DEBUG:teuthology.misc:Transferring archived files from smithi012:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674591/remote/smithi012/crash 2024-04-26T05:12:21.310 DEBUG:teuthology.orchestra.run.smithi012:> sudo tar c -f - -C /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/crash -- . 2024-04-26T05:12:21.363 INFO:teuthology.orchestra.run.smithi012.stderr:tar: /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/crash: Cannot open: No such file or directory 2024-04-26T05:12:21.363 INFO:teuthology.orchestra.run.smithi012.stderr:tar: Error is not recoverable: exiting now 2024-04-26T05:12:21.365 DEBUG:teuthology.misc:Transferring archived files from smithi116:/var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674591/remote/smithi116/crash 2024-04-26T05:12:21.366 DEBUG:teuthology.orchestra.run.smithi116:> sudo tar c -f - -C /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/crash -- . 2024-04-26T05:12:21.400 INFO:teuthology.orchestra.run.smithi116.stderr:tar: /var/lib/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/crash: Cannot open: No such file or directory 2024-04-26T05:12:21.400 INFO:teuthology.orchestra.run.smithi116.stderr:tar: Error is not recoverable: exiting now 2024-04-26T05:12:21.402 INFO:tasks.cephadm:Checking cluster log for badness... 2024-04-26T05:12:21.402 DEBUG:teuthology.orchestra.run.smithi012:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-26T05:12:21.433 INFO:teuthology.orchestra.run.smithi012.stdout:1714107238.8497784 mon.a (mon.0) 672 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T05:12:21.434 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-04-26T05:12:21.434 INFO:tasks.cephadm:Compressing logs... 2024-04-26T05:12:21.434 DEBUG:teuthology.orchestra.run.smithi012:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-26T05:12:21.477 DEBUG:teuthology.orchestra.run.smithi116:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-26T05:12:21.501 INFO:teuthology.orchestra.run.smithi116.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-26T05:12:21.501 INFO:teuthology.orchestra.run.smithi116.stderr:: No such file or directory 2024-04-26T05:12:21.501 INFO:teuthology.orchestra.run.smithi116.stderr:gzip -5 --verbose -- /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph.log 2024-04-26T05:12:21.501 INFO:teuthology.orchestra.run.smithi116.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph.audit.log 2024-04-26T05:12:21.501 INFO:teuthology.orchestra.run.smithi116.stderr:gzip -5 --verbose/var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph.log: -- /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-osd.7.log 2024-04-26T05:12:21.501 INFO:teuthology.orchestra.run.smithi116.stderr:/var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-osd.5.log 2024-04-26T05:12:21.502 INFO:teuthology.orchestra.run.smithi116.stderr:gzip -5 --verbose -- /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-osd.4.log 2024-04-26T05:12:21.502 INFO:teuthology.orchestra.run.smithi116.stderr:/var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-osd.7.log: /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-mgr.x.log 2024-04-26T05:12:21.511 INFO:teuthology.orchestra.run.smithi116.stderr:/var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-osd.6.log 2024-04-26T05:12:21.511 INFO:teuthology.orchestra.run.smithi116.stderr:/var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph.cephadm.log 2024-04-26T05:12:21.511 INFO:teuthology.orchestra.run.smithi116.stderr:/var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-osd.6.log: 87.4% -- replaced with /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph.log.gz 2024-04-26T05:12:21.511 INFO:teuthology.orchestra.run.smithi116.stderr:gzip -5 --verbose -- /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-mds.foo.smithi116.kduyef.log 2024-04-26T05:12:21.512 INFO:teuthology.orchestra.run.smithi116.stderr:/var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph.cephadm.log: 83.4% -- replaced with /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph.cephadm.log.gz 2024-04-26T05:12:21.512 INFO:teuthology.orchestra.run.smithi116.stderr: 89.5% -- replaced with /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph.audit.log.gz 2024-04-26T05:12:21.512 INFO:teuthology.orchestra.run.smithi116.stderr:gzip -5 --verbose -- /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-volume.log 2024-04-26T05:12:21.512 INFO:teuthology.orchestra.run.smithi116.stderr:/var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-mds.foo.smithi116.kduyef.log: gzip -5 --verbose -- /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-mon.b.log 62.7% -- replaced with /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-mds.foo.smithi116.kduyef.log.gz 2024-04-26T05:12:21.512 INFO:teuthology.orchestra.run.smithi116.stderr: 2024-04-26T05:12:21.513 INFO:teuthology.orchestra.run.smithi012.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-26T05:12:21.513 INFO:teuthology.orchestra.run.smithi012.stderr:: No such file or directory 2024-04-26T05:12:21.513 INFO:teuthology.orchestra.run.smithi012.stderr:gzip -5 --verbose -- /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph.log 2024-04-26T05:12:21.513 INFO:teuthology.orchestra.run.smithi012.stderr:gzip -5 --verbose -- /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-osd.2.log 2024-04-26T05:12:21.513 INFO:teuthology.orchestra.run.smithi012.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph.audit.log 2024-04-26T05:12:21.513 INFO:teuthology.orchestra.run.smithi012.stderr:/var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph.log: /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-osd.3.log 2024-04-26T05:12:21.513 INFO:teuthology.orchestra.run.smithi012.stderr:gzip -5 --verbose -- /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-osd.1.log 2024-04-26T05:12:21.514 INFO:teuthology.orchestra.run.smithi012.stderr:/var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph.audit.log: /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-mgr.y.log 2024-04-26T05:12:21.514 INFO:teuthology.orchestra.run.smithi012.stderr:/var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-osd.0.log 2024-04-26T05:12:21.514 INFO:teuthology.orchestra.run.smithi012.stderr:/var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-mon.c.log 2024-04-26T05:12:21.514 INFO:teuthology.orchestra.run.smithi012.stderr:/var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-osd.0.log: 92.6% -- replaced with /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph.log.gz 2024-04-26T05:12:21.514 INFO:teuthology.orchestra.run.smithi012.stderr: 93.9% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-26T05:12:21.514 INFO:teuthology.orchestra.run.smithi012.stderr:gzip -5 --verbose -- /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-mon.a.log 2024-04-26T05:12:21.514 INFO:teuthology.orchestra.run.smithi012.stderr:/var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-mon.c.log: gzip -5 --verbose -- /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph.cephadm.log 2024-04-26T05:12:21.515 INFO:teuthology.orchestra.run.smithi012.stderr:/var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-mon.a.log: 93.5% -- replaced with /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph.audit.log.gz 2024-04-26T05:12:21.515 INFO:teuthology.orchestra.run.smithi116.stderr:/var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-volume.log: /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-mon.b.log: 91.3% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-26T05:12:21.521 INFO:teuthology.orchestra.run.smithi116.stderr: 91.3% -- replaced with /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-volume.log.gz 2024-04-26T05:12:21.524 INFO:teuthology.orchestra.run.smithi012.stderr:gzip -5 --verbose -- /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-mds.foo.smithi012.fdqwmo.log 2024-04-26T05:12:21.525 INFO:teuthology.orchestra.run.smithi012.stderr:/var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph.cephadm.log: 90.1% -- replaced with /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph.cephadm.log.gz 2024-04-26T05:12:21.537 INFO:teuthology.orchestra.run.smithi012.stderr:gzip -5 --verbose -- /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-volume.log 2024-04-26T05:12:21.538 INFO:teuthology.orchestra.run.smithi012.stderr:/var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-mds.foo.smithi012.fdqwmo.log: 82.0% -- replaced with /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-mds.foo.smithi012.fdqwmo.log.gz 2024-04-26T05:12:21.559 INFO:teuthology.orchestra.run.smithi012.stderr:/var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-volume.log: 91.1% -- replaced with /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-volume.log.gz 2024-04-26T05:12:21.567 INFO:teuthology.orchestra.run.smithi116.stderr: 89.7% -- replaced with /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-mgr.x.log.gz 2024-04-26T05:12:21.838 INFO:teuthology.orchestra.run.smithi012.stderr: 91.9% -- replaced with /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-mon.c.log.gz 2024-04-26T05:12:21.845 INFO:teuthology.orchestra.run.smithi012.stderr: 88.7% -- replaced with /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-mgr.y.log.gz 2024-04-26T05:12:21.913 INFO:teuthology.orchestra.run.smithi116.stderr: 92.3% -- replaced with /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-mon.b.log.gz 2024-04-26T05:12:22.023 INFO:teuthology.orchestra.run.smithi116.stderr: 93.4% -- replaced with /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-osd.4.log.gz 2024-04-26T05:12:22.034 INFO:teuthology.orchestra.run.smithi116.stderr: 92.9% -- replaced with /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-osd.6.log.gz 2024-04-26T05:12:22.039 INFO:teuthology.orchestra.run.smithi116.stderr: 93.7% -- replaced with /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-osd.5.log.gz 2024-04-26T05:12:22.121 INFO:teuthology.orchestra.run.smithi116.stderr: 93.8% -- replaced with /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-osd.7.log.gz 2024-04-26T05:12:22.123 INFO:teuthology.orchestra.run.smithi116.stderr: 2024-04-26T05:12:22.124 INFO:teuthology.orchestra.run.smithi116.stderr:real 0m0.633s 2024-04-26T05:12:22.124 INFO:teuthology.orchestra.run.smithi116.stderr:user 0m2.637s 2024-04-26T05:12:22.124 INFO:teuthology.orchestra.run.smithi116.stderr:sys 0m0.107s 2024-04-26T05:12:22.127 INFO:teuthology.orchestra.run.smithi012.stderr: 93.5% -- replaced with /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-osd.3.log.gz 2024-04-26T05:12:22.133 INFO:teuthology.orchestra.run.smithi012.stderr: 93.3% -- replaced with /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-osd.0.log.gz 2024-04-26T05:12:22.269 INFO:teuthology.orchestra.run.smithi012.stderr: 93.3% -- replaced with /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-osd.1.log.gz 2024-04-26T05:12:22.485 INFO:teuthology.orchestra.run.smithi012.stderr: 94.9% -- replaced with /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-osd.2.log.gz 2024-04-26T05:12:22.545 INFO:teuthology.orchestra.run.smithi012.stderr: 90.6% -- replaced with /var/log/ceph/0b9350c6-0388-11ef-bc93-c7b262605968/ceph-mon.a.log.gz 2024-04-26T05:12:22.546 INFO:teuthology.orchestra.run.smithi012.stderr: 2024-04-26T05:12:22.546 INFO:teuthology.orchestra.run.smithi012.stderr:real 0m1.057s 2024-04-26T05:12:22.546 INFO:teuthology.orchestra.run.smithi012.stderr:user 0m4.619s 2024-04-26T05:12:22.546 INFO:teuthology.orchestra.run.smithi012.stderr:sys 0m0.157s 2024-04-26T05:12:22.548 INFO:tasks.cephadm:Archiving logs... 2024-04-26T05:12:22.548 DEBUG:teuthology.misc:Transferring archived files from smithi012:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674591/remote/smithi012/log 2024-04-26T05:12:22.549 DEBUG:teuthology.orchestra.run.smithi012:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-26T05:12:23.204 DEBUG:teuthology.misc:Transferring archived files from smithi116:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674591/remote/smithi116/log 2024-04-26T05:12:23.205 DEBUG:teuthology.orchestra.run.smithi116:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-26T05:12:23.640 INFO:tasks.cephadm:Removing cluster... 2024-04-26T05:12:23.640 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 --force 2024-04-26T05:12:24.145 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 0b9350c6-0388-11ef-bc93-c7b262605968 --force 2024-04-26T05:12:24.620 INFO:tasks.cephadm:Removing cephadm ... 2024-04-26T05:12:24.621 DEBUG:teuthology.orchestra.run.smithi012:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-26T05:12:24.637 DEBUG:teuthology.orchestra.run.smithi116:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-26T05:12:24.652 INFO:tasks.cephadm:Teardown complete 2024-04-26T05:12:24.653 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-04-26T05:12:24.667 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-04-26T05:12:24.668 DEBUG:teuthology.orchestra.run.smithi012:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-04-26T05:12:24.681 DEBUG:teuthology.orchestra.run.smithi116:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-04-26T05:12:24.768 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-04-26T05:12:24.777 INFO:teuthology.task.clock:Checking final clock skew... 2024-04-26T05:12:24.778 DEBUG:teuthology.orchestra.run.smithi012:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-26T05:12:24.781 DEBUG:teuthology.orchestra.run.smithi116:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-26T05:12:24.794 INFO:teuthology.orchestra.run.smithi012.stderr:bash: line 1: ntpq: command not found 2024-04-26T05:12:24.794 INFO:teuthology.orchestra.run.smithi116.stderr:bash: line 1: ntpq: command not found 2024-04-26T05:12:24.799 INFO:teuthology.orchestra.run.smithi012.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-26T05:12:24.801 INFO:teuthology.orchestra.run.smithi012.stdout:=============================================================================== 2024-04-26T05:12:24.801 INFO:teuthology.orchestra.run.smithi012.stdout:^* hv01.front.sepia.ceph.com 3 6 377 18 +939us[ +943us] +/- 55ms 2024-04-26T05:12:24.801 INFO:teuthology.orchestra.run.smithi012.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 20 -1342us[-1338us] +/- 75ms 2024-04-26T05:12:24.801 INFO:teuthology.orchestra.run.smithi012.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 17 -63us[ -63us] +/- 69ms 2024-04-26T05:12:24.801 INFO:teuthology.orchestra.run.smithi012.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T05:12:24.801 INFO:teuthology.orchestra.run.smithi116.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-26T05:12:24.802 INFO:teuthology.orchestra.run.smithi116.stdout:=============================================================================== 2024-04-26T05:12:24.802 INFO:teuthology.orchestra.run.smithi116.stdout:^* hv01.front.sepia.ceph.com 3 6 377 16 +974us[ +973us] +/- 55ms 2024-04-26T05:12:24.802 INFO:teuthology.orchestra.run.smithi116.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 17 -1307us[-1309us] +/- 75ms 2024-04-26T05:12:24.802 INFO:teuthology.orchestra.run.smithi116.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 17 -28us[ -29us] +/- 69ms 2024-04-26T05:12:24.802 INFO:teuthology.orchestra.run.smithi116.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T05:12:24.802 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-26T05:12:24.822 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-26T05:12:24.822 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-26T05:12:24.851 DEBUG:teuthology.orchestra.run.smithi012:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-26T05:12:24.936 DEBUG:teuthology.orchestra.run.smithi116:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-26T05:12:25.006 DEBUG:teuthology.orchestra.run.smithi012:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-26T05:12:25.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T05:12:25.035 DEBUG:teuthology.orchestra.run.smithi116:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-26T05:12:25.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T05:12:25.062 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-26T05:12:25.072 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-26T05:12:25.101 INFO:teuthology.task.internal:Duration was 1853.545808 seconds 2024-04-26T05:12:25.101 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-26T05:12:25.123 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-26T05:12:25.123 DEBUG:teuthology.orchestra.run.smithi012:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-26T05:12:25.126 DEBUG:teuthology.orchestra.run.smithi116:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-26T05:12:25.161 INFO:teuthology.orchestra.run.smithi116.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-26T05:12:25.162 INFO:teuthology.orchestra.run.smithi012.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-26T05:12:25.555 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-26T05:12:25.556 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi012.front.sepia.ceph.com 2024-04-26T05:12:25.556 DEBUG:teuthology.orchestra.run.smithi012:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-26T05:12:25.618 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi116.front.sepia.ceph.com 2024-04-26T05:12:25.619 DEBUG:teuthology.orchestra.run.smithi116:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-26T05:12:25.646 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-26T05:12:25.647 DEBUG:teuthology.orchestra.run.smithi012:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-26T05:12:25.661 DEBUG:teuthology.orchestra.run.smithi116:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-26T05:12:25.711 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-26T05:12:25.711 DEBUG:teuthology.orchestra.run.smithi012:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-26T05:12:25.728 DEBUG:teuthology.orchestra.run.smithi116:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-26T05:12:26.176 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-26T05:12:26.187 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-26T05:12:26.187 DEBUG:teuthology.orchestra.run.smithi012:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-26T05:12:26.211 DEBUG:teuthology.orchestra.run.smithi116:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-26T05:12:26.235 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-26T05:12:26.246 DEBUG:teuthology.orchestra.run.smithi012:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-26T05:12:26.254 DEBUG:teuthology.orchestra.run.smithi116:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-26T05:12:26.274 INFO:teuthology.orchestra.run.smithi012.stdout:kernel.core_pattern = core 2024-04-26T05:12:26.298 INFO:teuthology.orchestra.run.smithi116.stdout:kernel.core_pattern = core 2024-04-26T05:12:26.369 DEBUG:teuthology.orchestra.run.smithi012:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-26T05:12:26.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T05:12:26.384 DEBUG:teuthology.orchestra.run.smithi116:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-26T05:12:26.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T05:12:26.423 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-26T05:12:26.433 INFO:teuthology.task.internal:Transferring archived files... 2024-04-26T05:12:26.434 DEBUG:teuthology.misc:Transferring archived files from smithi012:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674591/remote/smithi012 2024-04-26T05:12:26.434 DEBUG:teuthology.orchestra.run.smithi012:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-26T05:12:26.491 DEBUG:teuthology.misc:Transferring archived files from smithi116:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674591/remote/smithi116 2024-04-26T05:12:26.491 DEBUG:teuthology.orchestra.run.smithi116:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-26T05:12:26.527 INFO:teuthology.task.internal:Removing archive directory... 2024-04-26T05:12:26.527 DEBUG:teuthology.orchestra.run.smithi012:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-26T05:12:26.529 DEBUG:teuthology.orchestra.run.smithi116:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-26T05:12:26.575 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-26T05:12:26.588 INFO:teuthology.task.internal:Not uploading archives. 2024-04-26T05:12:26.588 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-26T05:12:26.625 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-26T05:12:26.625 DEBUG:teuthology.orchestra.run.smithi012:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-26T05:12:26.627 DEBUG:teuthology.orchestra.run.smithi116:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-26T05:12:26.639 INFO:teuthology.orchestra.run.smithi012.stdout: 262213 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 26 05:12 /home/ubuntu/cephtest 2024-04-26T05:12:26.641 INFO:teuthology.orchestra.run.smithi116.stdout: 262213 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 26 05:12 /home/ubuntu/cephtest 2024-04-26T05:12:26.643 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-26T05:12:26.654 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-26T05:12:26.726 INFO:teuthology.nuke:Checking targets against current locks 2024-04-26T05:12:26.752 DEBUG:teuthology.nuke:shortname: smithi012 2024-04-26T05:12:26.752 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-26T05:12:26.773 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi012.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674591', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-26 04:35:51.182766', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLQp1eksonPbHSsFkISdJS1mr85/NCEJlpWQ7569jVuIEjfQ4+x+AcGk7ceUAMcYqkHkjRQfjcNAymm0jFgno8I='} 2024-04-26T05:12:26.776 DEBUG:teuthology.nuke:shortname: smithi116 2024-04-26T05:12:26.776 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-26T05:12:26.800 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi116.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674591', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-26 04:35:51.184183', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPcs/l1BxH4L87ucyG+asq1KBgaJ3eiUQ1VuYA1pzhZi8/EK1kYC2o5jiQ7S2SuSMTa9ZrbPD6jIhFj/bdLTNwM='} 2024-04-26T05:12:26.825 INFO:teuthology.orchestra.console.smithi012:Power off 2024-04-26T05:12:26.826 DEBUG:teuthology.orchestra.console.smithi012:pexpect command: ipmitool -H smithi012.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-26T05:12:26.852 DEBUG:teuthology.orchestra.console.smithi012:power off output: Chassis Power Control: Down/Off 2024-04-26T05:12:26.852 DEBUG:teuthology.orchestra.console.smithi012:pexpect command: ipmitool -H smithi012.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T05:12:26.868 INFO:teuthology.orchestra.console.smithi116:Power off 2024-04-26T05:12:26.868 DEBUG:teuthology.orchestra.console.smithi116:pexpect command: ipmitool -H smithi116.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-26T05:12:26.878 DEBUG:teuthology.orchestra.console.smithi012:check power output: Chassis Power is on 2024-04-26T05:12:26.891 DEBUG:teuthology.orchestra.console.smithi116:power off output: Chassis Power Control: Down/Off 2024-04-26T05:12:26.891 DEBUG:teuthology.orchestra.console.smithi116:pexpect command: ipmitool -H smithi116.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T05:12:26.916 DEBUG:teuthology.orchestra.console.smithi116:check power output: Chassis Power is on 2024-04-26T05:12:30.879 DEBUG:teuthology.orchestra.console.smithi012:pexpect command: ipmitool -H smithi012.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T05:12:30.917 DEBUG:teuthology.orchestra.console.smithi116:pexpect command: ipmitool -H smithi116.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T05:12:30.993 DEBUG:teuthology.orchestra.console.smithi012:check power output: Chassis Power is on 2024-04-26T05:12:31.028 DEBUG:teuthology.orchestra.console.smithi116:check power output: Chassis Power is on 2024-04-26T05:12:34.998 DEBUG:teuthology.orchestra.console.smithi012:pexpect command: ipmitool -H smithi012.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T05:12:35.028 DEBUG:teuthology.orchestra.console.smithi116:pexpect command: ipmitool -H smithi116.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T05:12:35.109 DEBUG:teuthology.orchestra.console.smithi012:check power output: Chassis Power is off 2024-04-26T05:12:35.139 DEBUG:teuthology.orchestra.console.smithi116:check power output: Chassis Power is off 2024-04-26T05:12:35.210 INFO:teuthology.orchestra.console.smithi012:Power off completed 2024-04-26T05:12:35.240 INFO:teuthology.orchestra.console.smithi116:Power off completed 2024-04-26T05:12:35.344 INFO:teuthology.run:Summary data: description: upgrade/quincy-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} duration: 1853.5458076000214 failure_reason: timeout expired in wait_until_healthy flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=6c5aaed31b5d4770a9b5b8c69e943148 status: fail success: false 2024-04-26T05:12:35.345 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-26T05:12:35.417 INFO:teuthology.run:FAIL